From 2639af6531bbf7f4175e4a39ac7b5cc1d1f0188b Mon Sep 17 00:00:00 2001 From: Toby <45889209+BagelHole@users.noreply.github.com> Date: Tue, 27 Jan 2026 17:35:45 -0500 Subject: [PATCH] . --- .gitattributes | 2 + CONTRIBUTING.md | 256 +++++++++ LICENSE | 21 + README.md | 404 ++++++++++++++ compliance/auditing/audit-logging/SKILL.md | 82 +++ compliance/auditing/aws-cloudtrail/SKILL.md | 64 +++ .../auditing/azure-monitor-audit/SKILL.md | 57 ++ compliance/auditing/gcp-audit-logs/SKILL.md | 71 +++ .../continuity/business-continuity/SKILL.md | 86 +++ .../continuity/disaster-recovery/SKILL.md | 73 +++ .../continuity/incident-management/SKILL.md | 90 +++ .../continuity/runbook-creation/SKILL.md | 96 ++++ .../frameworks/fedramp-compliance/SKILL.md | 71 +++ .../frameworks/gdpr-compliance/SKILL.md | 64 +++ .../frameworks/hipaa-compliance/SKILL.md | 74 +++ .../frameworks/iso27001-compliance/SKILL.md | 85 +++ .../frameworks/pci-dss-compliance/SKILL.md | 77 +++ .../frameworks/soc2-compliance/SKILL.md | 82 +++ compliance/governance/access-review/SKILL.md | 76 +++ .../governance/asset-inventory/SKILL.md | 74 +++ .../governance/change-management/SKILL.md | 81 +++ compliance/governance/policy-as-code/SKILL.md | 69 +++ .../governance/vendor-management/SKILL.md | 73 +++ devops/ci-cd/azure-devops/SKILL.md | 432 +++++++++++++++ devops/ci-cd/circleci/SKILL.md | 450 +++++++++++++++ devops/ci-cd/github-actions/SKILL.md | 344 ++++++++++++ .../references/workflow-patterns.md | 125 +++++ devops/ci-cd/gitlab-ci/SKILL.md | 315 +++++++++++ .../gitlab-ci/references/pipeline-patterns.md | 113 ++++ devops/ci-cd/jenkins/SKILL.md | 437 +++++++++++++++ .../jenkins/references/pipeline-syntax.md | 128 +++++ .../containers/container-registries/SKILL.md | 411 ++++++++++++++ devops/containers/docker-compose/SKILL.md | 458 ++++++++++++++++ .../references/compose-patterns.md | 124 +++++ devops/containers/docker-management/SKILL.md | 402 ++++++++++++++ .../references/docker-commands.md | 111 ++++ devops/containers/podman/SKILL.md | 381 +++++++++++++ devops/observability/alerting-oncall/SKILL.md | 498 +++++++++++++++++ devops/observability/datadog/SKILL.md | 463 ++++++++++++++++ .../references/datadog-integrations.md | 119 ++++ devops/observability/elk-stack/SKILL.md | 462 ++++++++++++++++ .../references/elasticsearch-queries.md | 139 +++++ devops/observability/loki-logging/SKILL.md | 455 +++++++++++++++ devops/observability/new-relic/SKILL.md | 447 +++++++++++++++ .../observability/prometheus-grafana/SKILL.md | 445 +++++++++++++++ .../assets/grafana-dashboard-template.json | 198 +++++++ .../assets/prometheus-config.yaml | 115 ++++ .../references/alerting-rules.md | 170 ++++++ .../references/promql-cheatsheet.md | 168 ++++++ .../scripts/backup-grafana.sh | 64 +++ .../scripts/prometheus-health-check.sh | 82 +++ devops/orchestration/argocd-gitops/SKILL.md | 445 +++++++++++++++ .../references/argocd-patterns.md | 126 +++++ devops/orchestration/helm-charts/SKILL.md | 445 +++++++++++++++ .../helm-charts/references/helm-commands.md | 121 ++++ devops/orchestration/kubernetes-ops/SKILL.md | 516 ++++++++++++++++++ .../assets/deployment-template.yaml | 166 ++++++ .../references/best-practices.md | 187 +++++++ .../references/troubleshooting-guide.md | 165 ++++++ .../scripts/cluster-health-check.sh | 78 +++ .../scripts/namespace-cleanup.sh | 75 +++ .../kubernetes-ops/scripts/pod-debug.sh | 71 +++ devops/orchestration/kustomize/SKILL.md | 454 +++++++++++++++ devops/orchestration/openshift/SKILL.md | 461 ++++++++++++++++ devops/release/blue-green-deploy/SKILL.md | 501 +++++++++++++++++ devops/release/feature-flags/SKILL.md | 472 ++++++++++++++++ devops/release/git-workflow/SKILL.md | 435 +++++++++++++++ devops/release/semantic-versioning/SKILL.md | 477 ++++++++++++++++ infrastructure/cloud-aws/aws-ec2/SKILL.md | 81 +++ .../aws-ec2/references/ec2-operations.md | 98 ++++ .../cloud-aws/aws-ecs-fargate/SKILL.md | 88 +++ infrastructure/cloud-aws/aws-iam/SKILL.md | 97 ++++ infrastructure/cloud-aws/aws-lambda/SKILL.md | 84 +++ infrastructure/cloud-aws/aws-rds/SKILL.md | 78 +++ infrastructure/cloud-aws/aws-s3/SKILL.md | 88 +++ infrastructure/cloud-aws/aws-vpc/SKILL.md | 84 +++ .../cloud-aws/cloudformation/SKILL.md | 93 ++++ .../cloudformation/references/cfn-syntax.md | 123 +++++ .../cloud-aws/terraform-aws/SKILL.md | 100 ++++ .../terraform-aws/assets/vpc-module.tf | 152 ++++++ .../references/best-practices.md | 185 +++++++ .../terraform-aws/scripts/tf-init.sh | 134 +++++ .../cloud-azure/arm-templates/SKILL.md | 58 ++ .../arm-templates/references/arm-syntax.md | 108 ++++ infrastructure/cloud-azure/azure-aks/SKILL.md | 62 +++ .../cloud-azure/azure-functions/SKILL.md | 54 ++ .../cloud-azure/azure-networking/SKILL.md | 60 ++ infrastructure/cloud-azure/azure-sql/SKILL.md | 58 ++ infrastructure/cloud-azure/azure-vms/SKILL.md | 45 ++ .../azure-vms/references/vm-operations.md | 99 ++++ .../cloud-azure/terraform-azure/SKILL.md | 59 ++ .../terraform-azure/assets/vnet-module.tf | 80 +++ .../terraform-azure/scripts/tf-init-azure.sh | 130 +++++ .../cloud-gcp/gcp-cloud-functions/SKILL.md | 49 ++ .../cloud-gcp/gcp-cloud-sql/SKILL.md | 49 ++ infrastructure/cloud-gcp/gcp-compute/SKILL.md | 42 ++ infrastructure/cloud-gcp/gcp-gke/SKILL.md | 55 ++ .../cloud-gcp/gcp-networking/SKILL.md | 59 ++ .../cloud-gcp/terraform-gcp/SKILL.md | 66 +++ .../terraform-gcp/assets/vpc-module.tf | 114 ++++ .../terraform-gcp/scripts/tf-init-gcp.sh | 130 +++++ .../databases/database-backups/SKILL.md | 70 +++ infrastructure/databases/mongodb/SKILL.md | 79 +++ infrastructure/databases/mysql/SKILL.md | 78 +++ infrastructure/databases/postgresql/SKILL.md | 68 +++ infrastructure/databases/redis/SKILL.md | 74 +++ infrastructure/networking/cdn-setup/SKILL.md | 59 ++ .../networking/dns-management/SKILL.md | 67 +++ .../networking/load-balancing/SKILL.md | 64 +++ .../networking/reverse-proxy/SKILL.md | 68 +++ .../networking/service-mesh/SKILL.md | 70 +++ .../servers/linux-administration/SKILL.md | 65 +++ .../servers/performance-tuning/SKILL.md | 69 +++ .../servers/ssh-configuration/SKILL.md | 76 +++ .../servers/systemd-services/SKILL.md | 76 +++ .../servers/user-management/SKILL.md | 69 +++ .../servers/windows-server/SKILL.md | 56 ++ .../storage/backup-recovery/SKILL.md | 63 +++ infrastructure/storage/block-storage/SKILL.md | 56 ++ infrastructure/storage/nfs-storage/SKILL.md | 67 +++ .../storage/object-storage/SKILL.md | 52 ++ security/hardening/cis-benchmarks/SKILL.md | 126 +++++ .../hardening/container-hardening/SKILL.md | 102 ++++ .../references/container-security.md | 105 ++++ .../hardening/kubernetes-hardening/SKILL.md | 129 +++++ security/hardening/linux-hardening/SKILL.md | 130 +++++ .../linux-hardening/assets/ssh-hardening.conf | 111 ++++ .../assets/sysctl-hardening.conf | 104 ++++ .../references/cis-checklist.md | 139 +++++ .../linux-hardening/scripts/audit-system.sh | 138 +++++ .../linux-hardening/scripts/harden-system.sh | 131 +++++ security/hardening/windows-hardening/SKILL.md | 98 ++++ security/network/firewall-config/SKILL.md | 84 +++ .../firewall-config/assets/iptables-rules.sh | 103 ++++ .../references/iptables-guide.md | 127 +++++ .../firewall-config/scripts/firewall-audit.sh | 91 +++ .../firewall-config/scripts/setup-ufw.sh | 80 +++ security/network/ssl-tls-management/SKILL.md | 96 ++++ security/network/vpn-setup/SKILL.md | 75 +++ security/network/waf-setup/SKILL.md | 79 +++ .../network/waf-setup/references/waf-rules.md | 120 ++++ security/network/zero-trust/SKILL.md | 86 +++ .../operations/incident-response/SKILL.md | 96 ++++ .../assets/incident-template.md | 155 ++++++ .../references/incident-playbook.md | 147 +++++ .../references/ioc-hunting.md | 149 +++++ .../scripts/collect-evidence.sh | 109 ++++ .../operations/penetration-testing/SKILL.md | 111 ++++ .../operations/security-automation/SKILL.md | 116 ++++ security/operations/threat-modeling/SKILL.md | 96 ++++ .../references/threat-model-template.md | 100 ++++ security/scanning/container-scanning/SKILL.md | 391 +++++++++++++ security/scanning/dast-scanning/SKILL.md | 399 ++++++++++++++ .../dast-scanning/references/dast-tools.md | 105 ++++ .../scanning/dependency-scanning/SKILL.md | 432 +++++++++++++++ security/scanning/sast-scanning/SKILL.md | 419 ++++++++++++++ .../sast-scanning/references/sast-tools.md | 106 ++++ .../scanning/vulnerability-scanning/SKILL.md | 411 ++++++++++++++ .../assets/github-action-scan.yaml | 94 ++++ .../assets/trivy-config.yaml | 70 +++ .../references/remediation-guide.md | 135 +++++ .../references/scanner-comparison.md | 93 ++++ .../scripts/scan-images.sh | 53 ++ .../scripts/scan-repo.sh | 81 +++ security/secrets/aws-secrets-manager/SKILL.md | 85 +++ .../references/secrets-patterns.md | 115 ++++ security/secrets/azure-keyvault/SKILL.md | 88 +++ security/secrets/gcp-secret-manager/SKILL.md | 81 +++ security/secrets/hashicorp-vault/SKILL.md | 384 +++++++++++++ .../assets/kubernetes-auth.yaml | 81 +++ .../hashicorp-vault/assets/vault-config.hcl | 62 +++ .../references/secrets-engines.md | 136 +++++ .../references/vault-policies.md | 130 +++++ .../hashicorp-vault/scripts/vault-backup.sh | 55 ++ .../hashicorp-vault/scripts/vault-init.sh | 78 +++ security/secrets/sops-encryption/SKILL.md | 100 ++++ 176 files changed, 27104 insertions(+) create mode 100644 .gitattributes create mode 100644 CONTRIBUTING.md create mode 100644 LICENSE create mode 100644 README.md create mode 100644 compliance/auditing/audit-logging/SKILL.md create mode 100644 compliance/auditing/aws-cloudtrail/SKILL.md create mode 100644 compliance/auditing/azure-monitor-audit/SKILL.md create mode 100644 compliance/auditing/gcp-audit-logs/SKILL.md create mode 100644 compliance/continuity/business-continuity/SKILL.md create mode 100644 compliance/continuity/disaster-recovery/SKILL.md create mode 100644 compliance/continuity/incident-management/SKILL.md create mode 100644 compliance/continuity/runbook-creation/SKILL.md create mode 100644 compliance/frameworks/fedramp-compliance/SKILL.md create mode 100644 compliance/frameworks/gdpr-compliance/SKILL.md create mode 100644 compliance/frameworks/hipaa-compliance/SKILL.md create mode 100644 compliance/frameworks/iso27001-compliance/SKILL.md create mode 100644 compliance/frameworks/pci-dss-compliance/SKILL.md create mode 100644 compliance/frameworks/soc2-compliance/SKILL.md create mode 100644 compliance/governance/access-review/SKILL.md create mode 100644 compliance/governance/asset-inventory/SKILL.md create mode 100644 compliance/governance/change-management/SKILL.md create mode 100644 compliance/governance/policy-as-code/SKILL.md create mode 100644 compliance/governance/vendor-management/SKILL.md create mode 100644 devops/ci-cd/azure-devops/SKILL.md create mode 100644 devops/ci-cd/circleci/SKILL.md create mode 100644 devops/ci-cd/github-actions/SKILL.md create mode 100644 devops/ci-cd/github-actions/references/workflow-patterns.md create mode 100644 devops/ci-cd/gitlab-ci/SKILL.md create mode 100644 devops/ci-cd/gitlab-ci/references/pipeline-patterns.md create mode 100644 devops/ci-cd/jenkins/SKILL.md create mode 100644 devops/ci-cd/jenkins/references/pipeline-syntax.md create mode 100644 devops/containers/container-registries/SKILL.md create mode 100644 devops/containers/docker-compose/SKILL.md create mode 100644 devops/containers/docker-compose/references/compose-patterns.md create mode 100644 devops/containers/docker-management/SKILL.md create mode 100644 devops/containers/docker-management/references/docker-commands.md create mode 100644 devops/containers/podman/SKILL.md create mode 100644 devops/observability/alerting-oncall/SKILL.md create mode 100644 devops/observability/datadog/SKILL.md create mode 100644 devops/observability/datadog/references/datadog-integrations.md create mode 100644 devops/observability/elk-stack/SKILL.md create mode 100644 devops/observability/elk-stack/references/elasticsearch-queries.md create mode 100644 devops/observability/loki-logging/SKILL.md create mode 100644 devops/observability/new-relic/SKILL.md create mode 100644 devops/observability/prometheus-grafana/SKILL.md create mode 100644 devops/observability/prometheus-grafana/assets/grafana-dashboard-template.json create mode 100644 devops/observability/prometheus-grafana/assets/prometheus-config.yaml create mode 100644 devops/observability/prometheus-grafana/references/alerting-rules.md create mode 100644 devops/observability/prometheus-grafana/references/promql-cheatsheet.md create mode 100644 devops/observability/prometheus-grafana/scripts/backup-grafana.sh create mode 100644 devops/observability/prometheus-grafana/scripts/prometheus-health-check.sh create mode 100644 devops/orchestration/argocd-gitops/SKILL.md create mode 100644 devops/orchestration/argocd-gitops/references/argocd-patterns.md create mode 100644 devops/orchestration/helm-charts/SKILL.md create mode 100644 devops/orchestration/helm-charts/references/helm-commands.md create mode 100644 devops/orchestration/kubernetes-ops/SKILL.md create mode 100644 devops/orchestration/kubernetes-ops/assets/deployment-template.yaml create mode 100644 devops/orchestration/kubernetes-ops/references/best-practices.md create mode 100644 devops/orchestration/kubernetes-ops/references/troubleshooting-guide.md create mode 100644 devops/orchestration/kubernetes-ops/scripts/cluster-health-check.sh create mode 100644 devops/orchestration/kubernetes-ops/scripts/namespace-cleanup.sh create mode 100644 devops/orchestration/kubernetes-ops/scripts/pod-debug.sh create mode 100644 devops/orchestration/kustomize/SKILL.md create mode 100644 devops/orchestration/openshift/SKILL.md create mode 100644 devops/release/blue-green-deploy/SKILL.md create mode 100644 devops/release/feature-flags/SKILL.md create mode 100644 devops/release/git-workflow/SKILL.md create mode 100644 devops/release/semantic-versioning/SKILL.md create mode 100644 infrastructure/cloud-aws/aws-ec2/SKILL.md create mode 100644 infrastructure/cloud-aws/aws-ec2/references/ec2-operations.md create mode 100644 infrastructure/cloud-aws/aws-ecs-fargate/SKILL.md create mode 100644 infrastructure/cloud-aws/aws-iam/SKILL.md create mode 100644 infrastructure/cloud-aws/aws-lambda/SKILL.md create mode 100644 infrastructure/cloud-aws/aws-rds/SKILL.md create mode 100644 infrastructure/cloud-aws/aws-s3/SKILL.md create mode 100644 infrastructure/cloud-aws/aws-vpc/SKILL.md create mode 100644 infrastructure/cloud-aws/cloudformation/SKILL.md create mode 100644 infrastructure/cloud-aws/cloudformation/references/cfn-syntax.md create mode 100644 infrastructure/cloud-aws/terraform-aws/SKILL.md create mode 100644 infrastructure/cloud-aws/terraform-aws/assets/vpc-module.tf create mode 100644 infrastructure/cloud-aws/terraform-aws/references/best-practices.md create mode 100644 infrastructure/cloud-aws/terraform-aws/scripts/tf-init.sh create mode 100644 infrastructure/cloud-azure/arm-templates/SKILL.md create mode 100644 infrastructure/cloud-azure/arm-templates/references/arm-syntax.md create mode 100644 infrastructure/cloud-azure/azure-aks/SKILL.md create mode 100644 infrastructure/cloud-azure/azure-functions/SKILL.md create mode 100644 infrastructure/cloud-azure/azure-networking/SKILL.md create mode 100644 infrastructure/cloud-azure/azure-sql/SKILL.md create mode 100644 infrastructure/cloud-azure/azure-vms/SKILL.md create mode 100644 infrastructure/cloud-azure/azure-vms/references/vm-operations.md create mode 100644 infrastructure/cloud-azure/terraform-azure/SKILL.md create mode 100644 infrastructure/cloud-azure/terraform-azure/assets/vnet-module.tf create mode 100644 infrastructure/cloud-azure/terraform-azure/scripts/tf-init-azure.sh create mode 100644 infrastructure/cloud-gcp/gcp-cloud-functions/SKILL.md create mode 100644 infrastructure/cloud-gcp/gcp-cloud-sql/SKILL.md create mode 100644 infrastructure/cloud-gcp/gcp-compute/SKILL.md create mode 100644 infrastructure/cloud-gcp/gcp-gke/SKILL.md create mode 100644 infrastructure/cloud-gcp/gcp-networking/SKILL.md create mode 100644 infrastructure/cloud-gcp/terraform-gcp/SKILL.md create mode 100644 infrastructure/cloud-gcp/terraform-gcp/assets/vpc-module.tf create mode 100644 infrastructure/cloud-gcp/terraform-gcp/scripts/tf-init-gcp.sh create mode 100644 infrastructure/databases/database-backups/SKILL.md create mode 100644 infrastructure/databases/mongodb/SKILL.md create mode 100644 infrastructure/databases/mysql/SKILL.md create mode 100644 infrastructure/databases/postgresql/SKILL.md create mode 100644 infrastructure/databases/redis/SKILL.md create mode 100644 infrastructure/networking/cdn-setup/SKILL.md create mode 100644 infrastructure/networking/dns-management/SKILL.md create mode 100644 infrastructure/networking/load-balancing/SKILL.md create mode 100644 infrastructure/networking/reverse-proxy/SKILL.md create mode 100644 infrastructure/networking/service-mesh/SKILL.md create mode 100644 infrastructure/servers/linux-administration/SKILL.md create mode 100644 infrastructure/servers/performance-tuning/SKILL.md create mode 100644 infrastructure/servers/ssh-configuration/SKILL.md create mode 100644 infrastructure/servers/systemd-services/SKILL.md create mode 100644 infrastructure/servers/user-management/SKILL.md create mode 100644 infrastructure/servers/windows-server/SKILL.md create mode 100644 infrastructure/storage/backup-recovery/SKILL.md create mode 100644 infrastructure/storage/block-storage/SKILL.md create mode 100644 infrastructure/storage/nfs-storage/SKILL.md create mode 100644 infrastructure/storage/object-storage/SKILL.md create mode 100644 security/hardening/cis-benchmarks/SKILL.md create mode 100644 security/hardening/container-hardening/SKILL.md create mode 100644 security/hardening/container-hardening/references/container-security.md create mode 100644 security/hardening/kubernetes-hardening/SKILL.md create mode 100644 security/hardening/linux-hardening/SKILL.md create mode 100644 security/hardening/linux-hardening/assets/ssh-hardening.conf create mode 100644 security/hardening/linux-hardening/assets/sysctl-hardening.conf create mode 100644 security/hardening/linux-hardening/references/cis-checklist.md create mode 100644 security/hardening/linux-hardening/scripts/audit-system.sh create mode 100644 security/hardening/linux-hardening/scripts/harden-system.sh create mode 100644 security/hardening/windows-hardening/SKILL.md create mode 100644 security/network/firewall-config/SKILL.md create mode 100644 security/network/firewall-config/assets/iptables-rules.sh create mode 100644 security/network/firewall-config/references/iptables-guide.md create mode 100644 security/network/firewall-config/scripts/firewall-audit.sh create mode 100644 security/network/firewall-config/scripts/setup-ufw.sh create mode 100644 security/network/ssl-tls-management/SKILL.md create mode 100644 security/network/vpn-setup/SKILL.md create mode 100644 security/network/waf-setup/SKILL.md create mode 100644 security/network/waf-setup/references/waf-rules.md create mode 100644 security/network/zero-trust/SKILL.md create mode 100644 security/operations/incident-response/SKILL.md create mode 100644 security/operations/incident-response/assets/incident-template.md create mode 100644 security/operations/incident-response/references/incident-playbook.md create mode 100644 security/operations/incident-response/references/ioc-hunting.md create mode 100644 security/operations/incident-response/scripts/collect-evidence.sh create mode 100644 security/operations/penetration-testing/SKILL.md create mode 100644 security/operations/security-automation/SKILL.md create mode 100644 security/operations/threat-modeling/SKILL.md create mode 100644 security/operations/threat-modeling/references/threat-model-template.md create mode 100644 security/scanning/container-scanning/SKILL.md create mode 100644 security/scanning/dast-scanning/SKILL.md create mode 100644 security/scanning/dast-scanning/references/dast-tools.md create mode 100644 security/scanning/dependency-scanning/SKILL.md create mode 100644 security/scanning/sast-scanning/SKILL.md create mode 100644 security/scanning/sast-scanning/references/sast-tools.md create mode 100644 security/scanning/vulnerability-scanning/SKILL.md create mode 100644 security/scanning/vulnerability-scanning/assets/github-action-scan.yaml create mode 100644 security/scanning/vulnerability-scanning/assets/trivy-config.yaml create mode 100644 security/scanning/vulnerability-scanning/references/remediation-guide.md create mode 100644 security/scanning/vulnerability-scanning/references/scanner-comparison.md create mode 100644 security/scanning/vulnerability-scanning/scripts/scan-images.sh create mode 100644 security/scanning/vulnerability-scanning/scripts/scan-repo.sh create mode 100644 security/secrets/aws-secrets-manager/SKILL.md create mode 100644 security/secrets/aws-secrets-manager/references/secrets-patterns.md create mode 100644 security/secrets/azure-keyvault/SKILL.md create mode 100644 security/secrets/gcp-secret-manager/SKILL.md create mode 100644 security/secrets/hashicorp-vault/SKILL.md create mode 100644 security/secrets/hashicorp-vault/assets/kubernetes-auth.yaml create mode 100644 security/secrets/hashicorp-vault/assets/vault-config.hcl create mode 100644 security/secrets/hashicorp-vault/references/secrets-engines.md create mode 100644 security/secrets/hashicorp-vault/references/vault-policies.md create mode 100644 security/secrets/hashicorp-vault/scripts/vault-backup.sh create mode 100644 security/secrets/hashicorp-vault/scripts/vault-init.sh create mode 100644 security/secrets/sops-encryption/SKILL.md diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..dfe0770 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +# Auto detect text files and perform LF normalization +* text=auto diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..bd1a55c --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,256 @@ +# Contributing to DevOps Security Agent Skills + +Thank you for your interest in contributing! This document provides guidelines for adding new skills or improving existing ones. + +## Table of Contents + +- [Getting Started](#getting-started) +- [Skill Structure](#skill-structure) +- [SKILL.md Template](#skillmd-template) +- [Naming Conventions](#naming-conventions) +- [Writing Guidelines](#writing-guidelines) +- [Submission Process](#submission-process) + +## Getting Started + +1. Fork this repository +2. Clone your fork locally +3. Create a new branch for your contribution +4. Make your changes +5. Submit a pull request + +## Skill Structure + +Each skill is a directory containing at minimum a `SKILL.md` file: + +``` +skill-name/ +├── SKILL.md # Required: instructions + metadata +├── scripts/ # Optional: executable code +├── references/ # Optional: detailed documentation +└── assets/ # Optional: templates, configs +``` + +### When to Include Additional Directories + +- **scripts/**: Include when the skill benefits from automation (validation scripts, setup helpers) +- **references/**: Include for detailed technical documentation that would bloat SKILL.md +- **assets/**: Include for templates, configuration files, or example resources + +## SKILL.md Template + +Use this template when creating new skills: + +```markdown +--- +name: skill-name +description: A clear description of what this skill does and when to use it. Include specific keywords that help agents identify relevant tasks. +license: MIT +metadata: + author: your-github-username + version: "1.0" +--- + +# Skill Title + +Brief introduction to the skill and its purpose. + +## When to Use This Skill + +Use this skill when: +- Condition 1 +- Condition 2 +- User mentions specific keywords or concepts + +## Prerequisites + +List any required: +- Tools or CLI utilities +- Access permissions +- Environment setup + +## Instructions + +### Task 1: Description + +Step-by-step instructions: + +1. First step +2. Second step +3. Third step + +Example: +\`\`\`bash +# Example command +command --flag value +\`\`\` + +### Task 2: Description + +Continue with additional tasks... + +## Common Issues + +### Issue 1 +**Problem**: Description of the issue +**Solution**: How to resolve it + +### Issue 2 +**Problem**: Description of the issue +**Solution**: How to resolve it + +## Best Practices + +- Practice 1 +- Practice 2 +- Practice 3 + +## Related Skills + +- [Related Skill 1](../related-skill-1/) +- [Related Skill 2](../related-skill-2/) +``` + +## Naming Conventions + +### Skill Names + +- Use lowercase letters, numbers, and hyphens only +- Maximum 64 characters +- Must not start or end with a hyphen +- Must not contain consecutive hyphens +- Directory name must match the `name` field in frontmatter + +**Good Examples:** +- `github-actions` +- `terraform-aws` +- `linux-hardening` + +**Bad Examples:** +- `GitHub-Actions` (uppercase not allowed) +- `-github-actions` (starts with hyphen) +- `github--actions` (consecutive hyphens) + +### File Names + +- Use lowercase with hyphens for markdown files +- Use lowercase with underscores for scripts +- Keep names descriptive but concise + +## Writing Guidelines + +### Description Field + +The description should: +- Be 1-1024 characters +- Explain what the skill does AND when to use it +- Include specific keywords for agent matching + +**Good:** +```yaml +description: Deploy and manage Docker containers including building images, optimizing Dockerfiles, managing volumes, and troubleshooting container issues. Use when working with Docker, containers, or containerization. +``` + +**Poor:** +```yaml +description: Helps with Docker. +``` + +### Instructions + +- Write clear, actionable steps +- Include code examples with proper syntax highlighting +- Explain the "why" not just the "how" +- Keep the main SKILL.md under 500 lines +- Move detailed reference material to separate files + +### Code Examples + +- Always test your examples before submitting +- Include comments explaining non-obvious commands +- Show both the command and expected output where helpful +- Use realistic but safe example values + +### Progressive Disclosure + +Structure skills for efficient context usage: + +1. **Frontmatter** (~100 tokens): Name and description only +2. **Main Instructions** (<5000 tokens): Core guidance in SKILL.md +3. **References** (as needed): Detailed docs in separate files + +## Domain Organization + +Place skills in the appropriate domain and category: + +``` +devops/ +├── ci-cd/ # CI/CD pipelines and automation +├── containers/ # Container management +├── orchestration/ # Kubernetes and orchestration +├── observability/ # Monitoring, logging, alerting +└── release/ # Release and deployment strategies + +security/ +├── scanning/ # Vulnerability and code scanning +├── secrets/ # Secrets management +├── hardening/ # System and container hardening +├── network/ # Network security +└── operations/ # Security operations + +infrastructure/ +├── cloud-aws/ # AWS services +├── cloud-azure/ # Azure services +├── cloud-gcp/ # GCP services +├── servers/ # Server management +├── networking/ # Network infrastructure +├── databases/ # Database management +└── storage/ # Storage solutions + +compliance/ +├── frameworks/ # Compliance frameworks +├── governance/ # Governance and policy +├── auditing/ # Audit and logging +└── continuity/ # Business continuity +``` + +## Submission Process + +### Before Submitting + +1. **Validate your skill** using skills-ref: + ```bash + skills-ref validate ./path/to/your-skill + ``` + +2. **Test your instructions** - ensure they work as documented + +3. **Check for duplicates** - ensure a similar skill doesn't already exist + +4. **Review the style** - match the conventions of existing skills + +### Pull Request Guidelines + +- Use a clear, descriptive title +- Reference any related issues +- Describe what the skill does and why it's useful +- Include any testing you've done + +### Review Criteria + +Submissions are reviewed for: + +- **Accuracy**: Instructions must be correct and tested +- **Clarity**: Easy to understand and follow +- **Completeness**: Covers common use cases and edge cases +- **Consistency**: Follows repository conventions +- **Value**: Adds meaningful capability for DevOps/Security tasks + +## Questions? + +Open an issue for: +- Questions about contributing +- Suggestions for new skills +- Feedback on existing skills + +Thank you for contributing! diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..67c5c5c --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Toby Miller + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..11a8714 --- /dev/null +++ b/README.md @@ -0,0 +1,404 @@ +
+ +# 🛡️ DevOps & Security Agent Skills + +### Your AI-Powered Second Brain for Infrastructure & Security + +*Stop Googling. Start Shipping.* + +[![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](LICENSE) +[![PRs Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg)](CONTRIBUTING.md) +[![Agent Skills](https://img.shields.io/badge/Format-Agent%20Skills-blueviolet.svg)](https://agentskills.io) + +
+ +**[Explore Skills](#skill-catalog)** · **[Get Started](#quick-start)** · **[Contribute](CONTRIBUTING.md)** + +
+ +   +   +   +   +   +   +   + + +
+ +--- + +## 💡 The Problem + +You're a **solo founder**, **indie hacker**, or **one-person DevOps team**. You need to: + +- Set up CI/CD pipelines across 5 different platforms +- Harden your Linux servers (but you forgot the sysctl parameters) +- Write that Terraform module for the 47th time +- Remember how CloudTrail works... again +- Configure Kubernetes security contexts properly +- Actually understand what SOC2 needs + +**You can't remember everything. You shouldn't have to.** + +--- + +## 🚀 The Solution + +This repo is a **comprehensive knowledge base** designed to be loaded into AI agents. It's your **DevOps second brain** — battle-tested scripts, production-ready configs, and expert knowledge organized using the [Agent Skills](https://agentskills.io) format: + +| Domain | What You Get | +|--------|--------------| +| 🔧 **DevOps** | CI/CD, containers, K8s, observability, release management | +| 🔒 **Security** | Scanning, secrets, hardening, network security, incident response | +| ☁️ **Infrastructure** | AWS, Azure, GCP, servers, networking, databases, storage | +| 📋 **Compliance** | SOC2, HIPAA, GDPR, PCI-DSS, governance, auditing | + +--- + +## ✨ What's Inside + +This isn't just documentation. Each skill includes: + +``` +skill/ +├── SKILL.md # AI-readable instructions & knowledge +├── scripts/ # Ready-to-run automation scripts +├── references/ # Deep-dive guides & cheatsheets +└── assets/ # Config templates & examples +``` + +### 🎯 Real Examples + +**Need to debug a crashing pod?** +```bash +./devops/orchestration/kubernetes-ops/scripts/pod-debug.sh my-pod +``` + +**Hardening a fresh Linux server?** +```bash +./security/hardening/linux-hardening/scripts/harden-system.sh --apply +``` + +**Setting up Vault from scratch?** +```bash +./security/secrets/hashicorp-vault/scripts/vault-init.sh +``` + +**Collecting evidence during an incident?** +```bash +./security/operations/incident-response/scripts/collect-evidence.sh INC-2024-001 +``` + +--- + +## 🧠 How It Works + +[Agent Skills](https://agentskills.io) is an open format for extending AI agent capabilities. Here's the flow: + +``` +┌─────────────────────────────────────────────────────────────────┐ +│ 1. DISCOVER 2. MATCH 3. ACTIVATE │ +│ │ +│ Agent scans → User asks about → Agent reads full │ +│ skill folders Kubernetes SKILL.md + runs │ +│ at startup debugging scripts as needed │ +└─────────────────────────────────────────────────────────────────┘ +``` + +Each `SKILL.md` has YAML frontmatter (name + description) that agents load at startup for matching, and markdown instructions that get loaded only when the skill is activated. This keeps context usage efficient. + +📖 **Full spec:** [agentskills.io/specification](https://agentskills.io/specification) + +--- + +## 🏃 Quick Start + +### 1. Download the Skills + +```bash +# Clone to your skills directory +git clone https://github.com/bagelhole/DevOps-Security-Agent-Skills.git ~/.skills/devops-security + +# Or add as a submodule to your project +git submodule add https://github.com/bagelhole/DevOps-Security-Agent-Skills.git .skills/devops-security +``` + +### 2. Integrate with Your Agent + +**Filesystem-based agents** (Cursor, Claude with computer use, Cline, etc.) are the easiest — the agent can read skills directly: + +```bash +# Agent reads skill when needed +cat ~/.skills/devops-security/devops/orchestration/kubernetes-ops/SKILL.md +``` + +**Tool-based agents** need skills injected into the system prompt. Use the [skills-ref](https://github.com/agentskills/agentskills/tree/main/skills-ref) CLI: + +```bash +# Generate XML for your agent's system prompt +skills-ref to-prompt ~/.skills/devops-security/devops/ci-cd/* + +# Output: +# +# +# github-actions +# Build, test, and deploy with GitHub Actions workflows... +# ~/.skills/devops-security/devops/ci-cd/github-actions/SKILL.md +# +# ... +# +``` + +### 3. Validate Skills (Optional) + +```bash +# Check skill format is correct +skills-ref validate ~/.skills/devops-security/security/secrets/hashicorp-vault +``` + +### For Humans + +No agent? No problem. Browse the skills, copy the scripts, use the configs. It's MIT licensed — go wild. + +--- + +## 📚 Skill Catalog + +
+🔧 DevOps + +### CI/CD +| Skill | Description | +|-------|-------------| +| [github-actions](devops/ci-cd/github-actions/) | Build, test, and deploy with GitHub Actions | +| [gitlab-ci](devops/ci-cd/gitlab-ci/) | GitLab CI/CD pipelines and runners | +| [jenkins](devops/ci-cd/jenkins/) | Jenkins pipelines and shared libraries | +| [azure-devops](devops/ci-cd/azure-devops/) | Azure Pipelines and release management | +| [circleci](devops/ci-cd/circleci/) | CircleCI workflows and orbs | + +### Containers +| Skill | Description | +|-------|-------------| +| [docker-management](devops/containers/docker-management/) | Docker images, multi-stage builds, optimization | +| [docker-compose](devops/containers/docker-compose/) | Multi-container applications | +| [podman](devops/containers/podman/) | Rootless container management | +| [container-registries](devops/containers/container-registries/) | ECR, ACR, GCR, Docker Hub | + +### Orchestration +| Skill | Description | +|-------|-------------| +| [kubernetes-ops](devops/orchestration/kubernetes-ops/) | Deploy, scale, troubleshoot K8s | +| [helm-charts](devops/orchestration/helm-charts/) | Helm chart development and deployment | +| [argocd-gitops](devops/orchestration/argocd-gitops/) | GitOps with ArgoCD | +| [kustomize](devops/orchestration/kustomize/) | Kubernetes manifest customization | +| [openshift](devops/orchestration/openshift/) | OpenShift cluster management | + +### Observability +| Skill | Description | +|-------|-------------| +| [prometheus-grafana](devops/observability/prometheus-grafana/) | Metrics and dashboards | +| [elk-stack](devops/observability/elk-stack/) | Elasticsearch, Logstash, Kibana | +| [loki-logging](devops/observability/loki-logging/) | Grafana Loki log aggregation | +| [datadog](devops/observability/datadog/) | Datadog monitoring and APM | +| [new-relic](devops/observability/new-relic/) | New Relic observability | +| [alerting-oncall](devops/observability/alerting-oncall/) | Alert rules and on-call rotations | + +### Release Management +| Skill | Description | +|-------|-------------| +| [git-workflow](devops/release/git-workflow/) | Branching strategies and PR workflows | +| [semantic-versioning](devops/release/semantic-versioning/) | Automated versioning and changelogs | +| [feature-flags](devops/release/feature-flags/) | LaunchDarkly, Unleash | +| [blue-green-deploy](devops/release/blue-green-deploy/) | Zero-downtime deployments | + +
+ +
+🔒 Security + +### Scanning +| Skill | Description | +|-------|-------------| +| [vulnerability-scanning](security/scanning/vulnerability-scanning/) | CVE scanning with Trivy, Grype | +| [sast-scanning](security/scanning/sast-scanning/) | Semgrep, CodeQL, SonarQube | +| [dast-scanning](security/scanning/dast-scanning/) | OWASP ZAP, Nuclei | +| [dependency-scanning](security/scanning/dependency-scanning/) | Snyk, Dependabot | +| [container-scanning](security/scanning/container-scanning/) | Image vulnerability scanning | + +### Secrets Management +| Skill | Description | +|-------|-------------| +| [hashicorp-vault](security/secrets/hashicorp-vault/) | Vault setup, policies, secrets engines | +| [aws-secrets-manager](security/secrets/aws-secrets-manager/) | AWS secrets and rotation | +| [azure-keyvault](security/secrets/azure-keyvault/) | Azure Key Vault | +| [gcp-secret-manager](security/secrets/gcp-secret-manager/) | GCP Secret Manager | +| [sops-encryption](security/secrets/sops-encryption/) | Mozilla SOPS | + +### Hardening +| Skill | Description | +|-------|-------------| +| [linux-hardening](security/hardening/linux-hardening/) | CIS benchmarks, sysctl, SSH | +| [windows-hardening](security/hardening/windows-hardening/) | Windows security baselines | +| [container-hardening](security/hardening/container-hardening/) | Secure Docker/K8s configs | +| [kubernetes-hardening](security/hardening/kubernetes-hardening/) | K8s security contexts and policies | +| [cis-benchmarks](security/hardening/cis-benchmarks/) | CIS benchmark auditing | + +### Network Security +| Skill | Description | +|-------|-------------| +| [firewall-config](security/network/firewall-config/) | iptables, UFW, cloud firewalls | +| [waf-setup](security/network/waf-setup/) | AWS WAF, Cloudflare WAF | +| [zero-trust](security/network/zero-trust/) | Zero-trust architecture | +| [vpn-setup](security/network/vpn-setup/) | WireGuard, OpenVPN | +| [ssl-tls-management](security/network/ssl-tls-management/) | Let's Encrypt, certificate management | + +### Security Operations +| Skill | Description | +|-------|-------------| +| [incident-response](security/operations/incident-response/) | IR playbooks and evidence collection | +| [threat-modeling](security/operations/threat-modeling/) | STRIDE methodology | +| [penetration-testing](security/operations/penetration-testing/) | Basic pentesting | +| [security-automation](security/operations/security-automation/) | Security workflow automation | + +
+ +
+☁️ Infrastructure + +### AWS +| Skill | Description | +|-------|-------------| +| [terraform-aws](infrastructure/cloud-aws/terraform-aws/) | AWS infrastructure as code | +| [cloudformation](infrastructure/cloud-aws/cloudformation/) | CloudFormation templates | +| [aws-ec2](infrastructure/cloud-aws/aws-ec2/) | EC2 instances and AMIs | +| [aws-ecs-fargate](infrastructure/cloud-aws/aws-ecs-fargate/) | Container orchestration | +| [aws-lambda](infrastructure/cloud-aws/aws-lambda/) | Serverless functions | +| [aws-rds](infrastructure/cloud-aws/aws-rds/) | Managed databases | +| [aws-s3](infrastructure/cloud-aws/aws-s3/) | Object storage | +| [aws-vpc](infrastructure/cloud-aws/aws-vpc/) | Networking | +| [aws-iam](infrastructure/cloud-aws/aws-iam/) | Identity and access | + +### Azure +| Skill | Description | +|-------|-------------| +| [terraform-azure](infrastructure/cloud-azure/terraform-azure/) | Azure infrastructure as code | +| [arm-templates](infrastructure/cloud-azure/arm-templates/) | ARM/Bicep templates | +| [azure-vms](infrastructure/cloud-azure/azure-vms/) | Virtual machines | +| [azure-functions](infrastructure/cloud-azure/azure-functions/) | Serverless | +| [azure-aks](infrastructure/cloud-azure/azure-aks/) | Kubernetes | +| [azure-sql](infrastructure/cloud-azure/azure-sql/) | Databases | +| [azure-networking](infrastructure/cloud-azure/azure-networking/) | VNets and NSGs | + +### GCP +| Skill | Description | +|-------|-------------| +| [terraform-gcp](infrastructure/cloud-gcp/terraform-gcp/) | GCP infrastructure as code | +| [gcp-compute](infrastructure/cloud-gcp/gcp-compute/) | Compute Engine | +| [gcp-cloud-functions](infrastructure/cloud-gcp/gcp-cloud-functions/) | Serverless | +| [gcp-gke](infrastructure/cloud-gcp/gcp-gke/) | Kubernetes | +| [gcp-cloud-sql](infrastructure/cloud-gcp/gcp-cloud-sql/) | Databases | +| [gcp-networking](infrastructure/cloud-gcp/gcp-networking/) | VPCs and firewall | + +### Server Management +| Skill | Description | +|-------|-------------| +| [linux-administration](infrastructure/servers/linux-administration/) | Core Linux admin | +| [windows-server](infrastructure/servers/windows-server/) | Windows administration | +| [ssh-configuration](infrastructure/servers/ssh-configuration/) | SSH and bastion hosts | +| [user-management](infrastructure/servers/user-management/) | Users, groups, sudo | +| [systemd-services](infrastructure/servers/systemd-services/) | Services and timers | +| [performance-tuning](infrastructure/servers/performance-tuning/) | System optimization | + +### Networking +| Skill | Description | +|-------|-------------| +| [dns-management](infrastructure/networking/dns-management/) | DNS and Route53 | +| [load-balancing](infrastructure/networking/load-balancing/) | ALB, nginx, HAProxy | +| [cdn-setup](infrastructure/networking/cdn-setup/) | CloudFront, Cloudflare | +| [reverse-proxy](infrastructure/networking/reverse-proxy/) | nginx, Traefik | +| [service-mesh](infrastructure/networking/service-mesh/) | Istio, Linkerd | + +### Databases +| Skill | Description | +|-------|-------------| +| [postgresql](infrastructure/databases/postgresql/) | PostgreSQL admin | +| [mysql](infrastructure/databases/mysql/) | MySQL/MariaDB | +| [mongodb](infrastructure/databases/mongodb/) | MongoDB clusters | +| [redis](infrastructure/databases/redis/) | Redis caching | +| [database-backups](infrastructure/databases/database-backups/) | Backup strategies | + +### Storage +| Skill | Description | +|-------|-------------| +| [block-storage](infrastructure/storage/block-storage/) | EBS, LVM | +| [object-storage](infrastructure/storage/object-storage/) | S3, MinIO | +| [nfs-storage](infrastructure/storage/nfs-storage/) | NFS servers | +| [backup-recovery](infrastructure/storage/backup-recovery/) | Backup with restic | + +
+ +
+📋 Compliance + +### Frameworks +| Skill | Description | +|-------|-------------| +| [soc2-compliance](compliance/frameworks/soc2-compliance/) | SOC2 Trust Services Criteria | +| [hipaa-compliance](compliance/frameworks/hipaa-compliance/) | HIPAA security rules | +| [gdpr-compliance](compliance/frameworks/gdpr-compliance/) | GDPR data protection | +| [pci-dss-compliance](compliance/frameworks/pci-dss-compliance/) | PCI-DSS requirements | +| [iso27001-compliance](compliance/frameworks/iso27001-compliance/) | ISO 27001 ISMS | +| [fedramp-compliance](compliance/frameworks/fedramp-compliance/) | FedRAMP controls | + +### Governance +| Skill | Description | +|-------|-------------| +| [policy-as-code](compliance/governance/policy-as-code/) | OPA, Kyverno, Checkov | +| [access-review](compliance/governance/access-review/) | IAM access reviews | +| [change-management](compliance/governance/change-management/) | Change control | +| [asset-inventory](compliance/governance/asset-inventory/) | Asset tracking | +| [vendor-management](compliance/governance/vendor-management/) | Third-party security | + +### Auditing +| Skill | Description | +|-------|-------------| +| [audit-logging](compliance/auditing/audit-logging/) | Centralized audit logs | +| [aws-cloudtrail](compliance/auditing/aws-cloudtrail/) | CloudTrail configuration | +| [azure-monitor-audit](compliance/auditing/azure-monitor-audit/) | Azure Monitor logs | +| [gcp-audit-logs](compliance/auditing/gcp-audit-logs/) | GCP Cloud Audit Logs | + +### Business Continuity +| Skill | Description | +|-------|-------------| +| [disaster-recovery](compliance/continuity/disaster-recovery/) | DR strategies | +| [business-continuity](compliance/continuity/business-continuity/) | BCP planning | +| [incident-management](compliance/continuity/incident-management/) | Incident processes | +| [runbook-creation](compliance/continuity/runbook-creation/) | Operational runbooks | + +
+ +--- + +## 🤝 Contributing + +Found a bug? Want to add a skill? PRs are welcome! + +See [CONTRIBUTING.md](CONTRIBUTING.md) for guidelines. + +--- + +## ⭐ Support + +If this helped you ship faster, **star this repo** — it helps others find it too. + +Built with ☕ by [Toby Miller](https://github.com/bagelhole) + +--- + +
+ +**[⬆ Back to Top](#-devops--security-agent-skills)** + +
diff --git a/compliance/auditing/audit-logging/SKILL.md b/compliance/auditing/audit-logging/SKILL.md new file mode 100644 index 0000000..a983d29 --- /dev/null +++ b/compliance/auditing/audit-logging/SKILL.md @@ -0,0 +1,82 @@ +--- +name: audit-logging +description: Implement centralized audit logging and SIEM integration. Configure log retention and security monitoring. Use when implementing audit trail requirements. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# Audit Logging + +Implement comprehensive audit logging for compliance. + +## Log Categories + +```yaml +audit_events: + authentication: + - Login attempts + - MFA events + - Session management + + authorization: + - Access grants + - Permission changes + - Role assignments + + data_access: + - Read operations + - Write operations + - Delete operations + + administrative: + - Configuration changes + - User management + - System changes +``` + +## Application Logging + +```python +import logging +import json + +class AuditLogger: + def log_event(self, event_type, user, resource, action, result): + log_entry = { + 'timestamp': datetime.utcnow().isoformat(), + 'event_type': event_type, + 'user': user, + 'resource': resource, + 'action': action, + 'result': result, + 'source_ip': request.remote_addr + } + logger.info(json.dumps(log_entry)) +``` + +## Centralized Logging + +```yaml +# Fluentd configuration + + @type tail + path /var/log/audit/*.log + tag audit.* + + + + @type elasticsearch + host elasticsearch.example.com + index_name audit-logs + +``` + +## Best Practices + +- Structured logging (JSON) +- Centralized collection +- Tamper-proof storage +- Retention policies +- Alerting on anomalies diff --git a/compliance/auditing/aws-cloudtrail/SKILL.md b/compliance/auditing/aws-cloudtrail/SKILL.md new file mode 100644 index 0000000..1a47115 --- /dev/null +++ b/compliance/auditing/aws-cloudtrail/SKILL.md @@ -0,0 +1,64 @@ +--- +name: aws-cloudtrail +description: Configure AWS CloudTrail for audit logging. Set up organization trails and event analysis. Use when auditing AWS activity. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# AWS CloudTrail + +Audit AWS account activity with CloudTrail. + +## Create Trail + +```bash +# Create organization trail +aws cloudtrail create-trail \ + --name org-audit-trail \ + --s3-bucket-name audit-logs-bucket \ + --is-organization-trail \ + --is-multi-region-trail \ + --enable-log-file-validation \ + --kms-key-id arn:aws:kms:... + +# Start logging +aws cloudtrail start-logging --name org-audit-trail +``` + +## Event Selectors + +```bash +# Log all management and data events +aws cloudtrail put-event-selectors \ + --trail-name org-audit-trail \ + --event-selectors '[{ + "ReadWriteType": "All", + "IncludeManagementEvents": true, + "DataResources": [{ + "Type": "AWS::S3::Object", + "Values": ["arn:aws:s3:::sensitive-bucket/"] + }] + }]' +``` + +## CloudTrail Lake + +```sql +-- Query events +SELECT eventTime, userIdentity.userName, eventName, sourceIPAddress +FROM cloudtrail_logs +WHERE eventTime > '2024-01-01' + AND eventName LIKE '%Delete%' +ORDER BY eventTime DESC +LIMIT 100 +``` + +## Best Practices + +- Organization-wide trails +- Enable log file validation +- Encrypt with KMS +- CloudWatch Logs integration +- Event alerting diff --git a/compliance/auditing/azure-monitor-audit/SKILL.md b/compliance/auditing/azure-monitor-audit/SKILL.md new file mode 100644 index 0000000..b62a6c9 --- /dev/null +++ b/compliance/auditing/azure-monitor-audit/SKILL.md @@ -0,0 +1,57 @@ +--- +name: azure-monitor-audit +description: Configure Azure Monitor and Activity Log for auditing. Set up diagnostic settings and log analytics. Use when auditing Azure activity. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# Azure Monitor Audit + +Audit Azure activity with Monitor and Activity Logs. + +## Diagnostic Settings + +```bash +# Enable diagnostic settings +az monitor diagnostic-settings create \ + --name audit-logs \ + --resource /subscriptions/{sub}/resourceGroups/{rg}/providers/... \ + --logs '[{"category":"AuditEvent","enabled":true}]' \ + --workspace /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.OperationalInsights/workspaces/{workspace} +``` + +## Activity Log Export + +```bash +# Export activity log to Log Analytics +az monitor diagnostic-settings subscription create \ + --name activity-log-export \ + --location global \ + --logs '[{"category":"Administrative","enabled":true},{"category":"Security","enabled":true}]' \ + --workspace /subscriptions/.../workspaces/audit-workspace +``` + +## Log Analytics Queries + +```kusto +// Failed login attempts +AuditLogs +| where TimeGenerated > ago(24h) +| where ResultType != "0" +| project TimeGenerated, Identity, ResultDescription, IPAddress + +// Administrative changes +AzureActivity +| where CategoryValue == "Administrative" +| where OperationNameValue contains "write" or OperationNameValue contains "delete" +| project TimeGenerated, Caller, OperationNameValue, ResourceGroup +``` + +## Best Practices + +- Centralize to Log Analytics +- Long-term archive to Storage +- Configure alerts +- Regular query reviews diff --git a/compliance/auditing/gcp-audit-logs/SKILL.md b/compliance/auditing/gcp-audit-logs/SKILL.md new file mode 100644 index 0000000..8a7e504 --- /dev/null +++ b/compliance/auditing/gcp-audit-logs/SKILL.md @@ -0,0 +1,71 @@ +--- +name: gcp-audit-logs +description: Configure GCP Cloud Audit Logs for compliance. Set up log routing and BigQuery analysis. Use when auditing GCP activity. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# GCP Audit Logs + +Audit GCP activity with Cloud Audit Logs. + +## Audit Log Types + +```yaml +log_types: + admin_activity: + - Always enabled + - API calls that modify resources + - No charge + + data_access: + - Must be enabled + - Read/write data operations + - Can be high volume + + system_event: + - Always enabled + - GCP system actions + + policy_denied: + - Always enabled + - Access denials +``` + +## Enable Data Access Logs + +```bash +# Enable for all services +gcloud logging sinks create audit-sink \ + storage.googleapis.com/audit-logs-bucket \ + --log-filter='logName:"cloudaudit.googleapis.com"' + +# IAM policy for data access logs +gcloud projects get-iam-policy PROJECT_ID > policy.yaml +# Add auditConfigs section +gcloud projects set-iam-policy PROJECT_ID policy.yaml +``` + +## BigQuery Analysis + +```sql +-- Query audit logs from BigQuery export +SELECT + timestamp, + protopayload_auditlog.authenticationInfo.principalEmail, + protopayload_auditlog.methodName, + resource.labels.project_id +FROM `project.dataset.cloudaudit_googleapis_com_activity_*` +WHERE timestamp > TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 7 DAY) + AND protopayload_auditlog.methodName LIKE '%delete%' +ORDER BY timestamp DESC +``` + +## Best Practices + +- Export to BigQuery for analysis +- Configure log retention +- Enable data access logs for sensitive resources +- Set up alerting policies diff --git a/compliance/continuity/business-continuity/SKILL.md b/compliance/continuity/business-continuity/SKILL.md new file mode 100644 index 0000000..579c1c6 --- /dev/null +++ b/compliance/continuity/business-continuity/SKILL.md @@ -0,0 +1,86 @@ +--- +name: business-continuity +description: Develop business continuity plans and impact analysis. Implement BCP testing and communication procedures. Use when building organizational resilience. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# Business Continuity Planning + +Develop and maintain business continuity capabilities. + +## BCP Framework + +```yaml +bcp_phases: + 1_analysis: + - Business Impact Analysis (BIA) + - Risk assessment + - Critical process identification + + 2_planning: + - Recovery strategies + - Resource requirements + - Communication plans + + 3_implementation: + - Procedure documentation + - Training + - Technology setup + + 4_testing: + - Plan exercises + - Gap identification + - Continuous improvement +``` + +## Business Impact Analysis + +```yaml +process_classification: + critical: + max_downtime: 4 hours + examples: Payment processing, authentication + + essential: + max_downtime: 24 hours + examples: Customer support, reporting + + necessary: + max_downtime: 72 hours + examples: Internal tools, analytics + + desirable: + max_downtime: 7 days + examples: Development environments +``` + +## Communication Plan + +```yaml +communication: + internal: + - Executive notification + - Team communication + - Status updates + + external: + - Customer notification + - Regulatory reporting + - Media relations + + channels: + - Primary: Slack/Teams + - Secondary: Email + - Emergency: Phone tree +``` + +## Best Practices + +- Annual BIA updates +- Regular plan testing +- Clear roles and responsibilities +- Multiple communication channels +- Executive sponsorship diff --git a/compliance/continuity/disaster-recovery/SKILL.md b/compliance/continuity/disaster-recovery/SKILL.md new file mode 100644 index 0000000..3ee6901 --- /dev/null +++ b/compliance/continuity/disaster-recovery/SKILL.md @@ -0,0 +1,73 @@ +--- +name: disaster-recovery +description: Implement disaster recovery strategies and runbooks. Configure RPO/RTO targets and failover procedures. Use when planning for business continuity. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# Disaster Recovery + +Implement disaster recovery strategies and procedures. + +## DR Metrics + +```yaml +recovery_metrics: + RTO: Recovery Time Objective + - Maximum acceptable downtime + - How long to restore service + + RPO: Recovery Point Objective + - Maximum acceptable data loss + - How much data can be lost +``` + +## DR Strategies + +| Strategy | RTO | RPO | Cost | +|----------|-----|-----|------| +| Backup & Restore | Hours | Hours | $ | +| Pilot Light | Minutes-Hours | Minutes | $$ | +| Warm Standby | Minutes | Seconds | $$$ | +| Multi-Site Active | Near-zero | Near-zero | $$$$ | + +## AWS Multi-Region + +```bash +# Cross-region RDS replica +aws rds create-db-instance-read-replica \ + --db-instance-identifier dr-replica \ + --source-db-instance-identifier prod-db \ + --source-region us-east-1 \ + --region us-west-2 + +# S3 cross-region replication +aws s3api put-bucket-replication \ + --bucket source-bucket \ + --replication-configuration file://replication.json +``` + +## DR Testing + +```yaml +dr_test_schedule: + tabletop: Quarterly + component_failover: Monthly + full_failover: Annually + +test_checklist: + - [ ] Verify backup integrity + - [ ] Test failover procedures + - [ ] Validate data consistency + - [ ] Measure actual RTO/RPO + - [ ] Document lessons learned +``` + +## Best Practices + +- Regular DR testing +- Automate failover where possible +- Document all procedures +- Update runbooks after tests diff --git a/compliance/continuity/incident-management/SKILL.md b/compliance/continuity/incident-management/SKILL.md new file mode 100644 index 0000000..a987d80 --- /dev/null +++ b/compliance/continuity/incident-management/SKILL.md @@ -0,0 +1,90 @@ +--- +name: incident-management +description: Implement incident management processes and escalation procedures. Configure on-call schedules and post-incident reviews. Use when managing production incidents. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# Incident Management + +Implement effective incident management processes. + +## Incident Severity + +| Severity | Impact | Response | Example | +|----------|--------|----------|---------| +| SEV1 | Total outage | Immediate, all-hands | Site down | +| SEV2 | Major degradation | Urgent, on-call | Feature broken | +| SEV3 | Minor impact | Standard | Slow performance | +| SEV4 | Minimal | Next business day | Cosmetic issue | + +## Incident Process + +```yaml +incident_workflow: + 1_detect: + - Alerting triggers + - Customer reports + - Monitoring anomalies + + 2_triage: + - Severity assessment + - Impact determination + - Team notification + + 3_respond: + - Incident commander assigned + - Communication established + - Mitigation started + + 4_resolve: + - Root cause addressed + - Service restored + - Customer notified + + 5_review: + - Timeline documented + - Root cause analysis + - Action items created +``` + +## Incident Commander + +```yaml +ic_responsibilities: + - Own incident resolution + - Coordinate response teams + - Manage communication + - Make escalation decisions + - Schedule post-mortem +``` + +## Post-Incident Review + +```markdown +## Incident Summary +- Duration: +- Impact: +- Severity: + +## Timeline + +## Root Cause + +## What Went Well + +## What Could Be Improved + +## Action Items +| Item | Owner | Due Date | +``` + +## Best Practices + +- Clear severity definitions +- Defined escalation paths +- Blameless post-mortems +- Action item tracking +- Regular training diff --git a/compliance/continuity/runbook-creation/SKILL.md b/compliance/continuity/runbook-creation/SKILL.md new file mode 100644 index 0000000..af25d01 --- /dev/null +++ b/compliance/continuity/runbook-creation/SKILL.md @@ -0,0 +1,96 @@ +--- +name: runbook-creation +description: Create operational runbooks and standard operating procedures. Document troubleshooting guides and recovery procedures. Use when documenting operational knowledge. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# Runbook Creation + +Create effective operational runbooks and procedures. + +## Runbook Structure + +```markdown +# Runbook: [Service/Process Name] + +## Overview +Brief description of the service and runbook purpose. + +## Prerequisites +- Required access +- Tools needed +- Knowledge required + +## Procedure +Step-by-step instructions with commands. + +## Verification +How to confirm success. + +## Rollback +Steps to undo if needed. + +## Escalation +When and how to escalate. + +## Related Runbooks +Links to related procedures. +``` + +## Example Runbook + +```markdown +# Runbook: Database Failover + +## Overview +Procedure to failover PostgreSQL to replica. + +## Prerequisites +- [ ] DBA access to primary and replica +- [ ] VPN connected +- [ ] Slack channel #db-ops open + +## Procedure + +### 1. Verify Replica Status +\`\`\`bash +psql -h replica -c "SELECT pg_is_in_recovery();" +# Should return 't' +\`\`\` + +### 2. Stop Application Writes +\`\`\`bash +kubectl scale deployment app --replicas=0 +\`\`\` + +### 3. Promote Replica +\`\`\`bash +psql -h replica -c "SELECT pg_promote();" +\`\`\` + +### 4. Update DNS +\`\`\`bash +aws route53 change-resource-record-sets ... +\`\`\` + +## Verification +- [ ] Application connects to new primary +- [ ] No replication lag errors +- [ ] Transactions completing + +## Escalation +If issues persist after 15 minutes, escalate to: +- Primary: @dba-lead +- Secondary: @platform-oncall +``` + +## Best Practices + +- Keep procedures simple and clear +- Include verification steps +- Test runbooks regularly +- Version control runbooks +- Include troubleshooting tips diff --git a/compliance/frameworks/fedramp-compliance/SKILL.md b/compliance/frameworks/fedramp-compliance/SKILL.md new file mode 100644 index 0000000..cb5b871 --- /dev/null +++ b/compliance/frameworks/fedramp-compliance/SKILL.md @@ -0,0 +1,71 @@ +--- +name: fedramp-compliance +description: Implement FedRAMP requirements for federal cloud services. Configure NIST 800-53 controls and continuous monitoring. Use when providing cloud services to US federal agencies. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# FedRAMP Compliance + +Implement FedRAMP requirements for federal cloud services. + +## Impact Levels + +```yaml +levels: + low: + controls: ~125 + use_case: Public data + + moderate: + controls: ~325 + use_case: CUI, most federal systems + + high: + controls: ~425 + use_case: Law enforcement, emergency services +``` + +## NIST 800-53 Families + +```yaml +control_families: + AC: Access Control + AU: Audit and Accountability + AT: Awareness and Training + CM: Configuration Management + CP: Contingency Planning + IA: Identification and Authentication + IR: Incident Response + MA: Maintenance + MP: Media Protection + PE: Physical Protection + PL: Planning + PS: Personnel Security + RA: Risk Assessment + CA: Assessment and Authorization + SC: System and Communications Protection + SI: System and Information Integrity + SA: System and Services Acquisition + PM: Program Management +``` + +## Continuous Monitoring + +```yaml +conmon: + vulnerability_scans: Monthly + penetration_tests: Annual + poa_m_updates: Monthly + security_assessment: Annual +``` + +## Best Practices + +- 3PAO assessment +- SSP documentation +- POA&M tracking +- Continuous monitoring +- Annual authorization diff --git a/compliance/frameworks/gdpr-compliance/SKILL.md b/compliance/frameworks/gdpr-compliance/SKILL.md new file mode 100644 index 0000000..a487943 --- /dev/null +++ b/compliance/frameworks/gdpr-compliance/SKILL.md @@ -0,0 +1,64 @@ +--- +name: gdpr-compliance +description: Implement GDPR data protection requirements. Configure consent management, data subject rights, and privacy by design. Use when processing EU personal data. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# GDPR Compliance + +Implement GDPR requirements for EU data protection. + +## Key Principles + +```yaml +principles: + lawfulness: Legal basis for processing + purpose_limitation: Specific, explicit purposes + data_minimization: Adequate, relevant, limited + accuracy: Accurate and up to date + storage_limitation: No longer than necessary + integrity: Secure processing + accountability: Demonstrate compliance +``` + +## Data Subject Rights + +```yaml +rights: + - Right to access + - Right to rectification + - Right to erasure + - Right to restrict processing + - Right to data portability + - Right to object + - Rights related to automated decisions +``` + +## Technical Implementation + +```python +# Data export for portability +def export_user_data(user_id): + return { + "profile": get_profile(user_id), + "activity": get_activity_log(user_id), + "preferences": get_preferences(user_id) + } + +# Right to erasure +def delete_user_data(user_id): + anonymize_profile(user_id) + delete_activity_log(user_id) + log_deletion(user_id) +``` + +## Best Practices + +- Privacy impact assessments +- Data processing agreements +- Consent management +- Breach notification (72 hours) +- Data Protection Officer (if required) diff --git a/compliance/frameworks/hipaa-compliance/SKILL.md b/compliance/frameworks/hipaa-compliance/SKILL.md new file mode 100644 index 0000000..8ff610f --- /dev/null +++ b/compliance/frameworks/hipaa-compliance/SKILL.md @@ -0,0 +1,74 @@ +--- +name: hipaa-compliance +description: Implement HIPAA security and privacy rules. Configure PHI protections and BAA requirements. Use when handling healthcare data. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# HIPAA Compliance + +Implement HIPAA requirements for healthcare data protection. + +## HIPAA Rules + +```yaml +security_rule: + administrative: + - Risk analysis + - Security management + - Workforce training + - Contingency planning + + physical: + - Facility access + - Workstation security + - Device controls + + technical: + - Access control + - Audit controls + - Integrity controls + - Transmission security +``` + +## Technical Safeguards + +```yaml +requirements: + encryption: + at_rest: AES-256 + in_transit: TLS 1.2+ + + access_control: + - Unique user IDs + - Emergency access procedure + - Automatic logoff + - Encryption/decryption + + audit: + - Access logging + - Activity monitoring + - Log retention (6 years) +``` + +## AWS HIPAA Setup + +```bash +# Enable CloudTrail for HIPAA auditing +aws cloudtrail create-trail \ + --name hipaa-audit-trail \ + --s3-bucket-name hipaa-logs \ + --is-multi-region-trail \ + --enable-log-file-validation + +# Use HIPAA-eligible services only +``` + +## Best Practices + +- Business Associate Agreements (BAAs) +- Minimum necessary access +- Breach notification procedures +- Regular risk assessments diff --git a/compliance/frameworks/iso27001-compliance/SKILL.md b/compliance/frameworks/iso27001-compliance/SKILL.md new file mode 100644 index 0000000..3436bb1 --- /dev/null +++ b/compliance/frameworks/iso27001-compliance/SKILL.md @@ -0,0 +1,85 @@ +--- +name: iso27001-compliance +description: Implement ISO 27001 Information Security Management System. Configure ISMS controls and risk management. Use when implementing enterprise security frameworks. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# ISO 27001 Compliance + +Implement ISO 27001 Information Security Management System. + +## ISMS Framework + +```yaml +plan_do_check_act: + plan: + - Define scope + - Risk assessment + - Risk treatment plan + - Statement of Applicability + + do: + - Implement controls + - Security awareness + - Document procedures + + check: + - Internal audits + - Management review + - Performance measurement + + act: + - Corrective actions + - Continual improvement +``` + +## Annex A Controls + +```yaml +control_domains: + A.5: Information security policies + A.6: Organization of information security + A.7: Human resource security + A.8: Asset management + A.9: Access control + A.10: Cryptography + A.11: Physical security + A.12: Operations security + A.13: Communications security + A.14: System acquisition/development + A.15: Supplier relationships + A.16: Incident management + A.17: Business continuity + A.18: Compliance +``` + +## Risk Assessment + +```yaml +risk_assessment: + identify: + - Asset inventory + - Threat identification + - Vulnerability assessment + + analyze: + - Likelihood rating + - Impact rating + - Risk calculation + + evaluate: + - Risk acceptance criteria + - Prioritization + - Treatment options +``` + +## Best Practices + +- Management commitment +- Risk-based approach +- Document everything +- Regular internal audits +- Continuous improvement diff --git a/compliance/frameworks/pci-dss-compliance/SKILL.md b/compliance/frameworks/pci-dss-compliance/SKILL.md new file mode 100644 index 0000000..c8a47a7 --- /dev/null +++ b/compliance/frameworks/pci-dss-compliance/SKILL.md @@ -0,0 +1,77 @@ +--- +name: pci-dss-compliance +description: Implement PCI DSS requirements for payment card data. Configure cardholder data environment and security controls. Use when processing payment cards. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# PCI DSS Compliance + +Implement PCI DSS requirements for payment card security. + +## Requirements + +```yaml +requirements: + 1_firewall: + - Network segmentation + - Firewall configuration + - CDE isolation + + 3_protect_data: + - Mask PAN display + - Encrypt stored data + - Key management + + 6_secure_systems: + - Patch management + - Secure development + - Change control + + 8_access_control: + - Unique IDs + - MFA for remote access + - Password policies + + 10_logging: + - Audit trail + - Time synchronization + - Log retention (1 year) + + 11_testing: + - Vulnerability scans + - Penetration testing + - IDS/IPS monitoring +``` + +## Network Segmentation + +``` +Internet --> DMZ --> Firewall --> CDE + | + Non-CDE <-- Firewall -- +``` + +## Data Protection + +```yaml +encryption: + at_rest: AES-256 + in_transit: TLS 1.2+ + key_storage: HSM or dedicated key vault + +tokenization: + - Replace PAN with token + - Store mapping securely + - Reduce CDE scope +``` + +## Best Practices + +- Minimize CDE scope +- Use tokenization +- Quarterly vulnerability scans +- Annual penetration tests +- ASV scan certification diff --git a/compliance/frameworks/soc2-compliance/SKILL.md b/compliance/frameworks/soc2-compliance/SKILL.md new file mode 100644 index 0000000..38d4cfd --- /dev/null +++ b/compliance/frameworks/soc2-compliance/SKILL.md @@ -0,0 +1,82 @@ +--- +name: soc2-compliance +description: Implement SOC 2 Trust Services Criteria. Configure security, availability, and processing integrity controls. Use when achieving SOC 2 certification. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# SOC 2 Compliance + +Implement SOC 2 Trust Services Criteria for certification. + +## Trust Services Criteria + +```yaml +criteria: + security: + - Access controls + - Change management + - Risk assessment + - Incident response + + availability: + - System monitoring + - Disaster recovery + - Capacity planning + - SLA management + + processing_integrity: + - Input validation + - Processing completeness + - Output accuracy + + confidentiality: + - Data classification + - Encryption + - Access restrictions + + privacy: + - Data collection notice + - Consent management + - Data retention +``` + +## Key Controls + +```yaml +controls: + CC6.1_logical_access: + - MFA enforcement + - Role-based access + - Access reviews + + CC7.2_monitoring: + - Log aggregation + - Alert thresholds + - Incident tracking + + CC8.1_change_management: + - Change requests + - Approval workflows + - Testing requirements +``` + +## Evidence Collection + +```bash +# Access review export +aws iam generate-credential-report +aws iam get-credential-report + +# Audit logs +aws cloudtrail lookup-events --start-time $(date -d '30 days ago' --iso) +``` + +## Best Practices + +- Continuous compliance monitoring +- Annual risk assessments +- Regular control testing +- Documentation maintenance diff --git a/compliance/governance/access-review/SKILL.md b/compliance/governance/access-review/SKILL.md new file mode 100644 index 0000000..f385cd3 --- /dev/null +++ b/compliance/governance/access-review/SKILL.md @@ -0,0 +1,76 @@ +--- +name: access-review +description: Conduct periodic access reviews and certifications. Implement access governance and recertification workflows. Use when managing access compliance. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# Access Review + +Implement periodic access review processes. + +## Review Process + +```yaml +access_review_workflow: + 1_extract: + - Pull access data from systems + - Generate access report + + 2_review: + - Manager certification + - Risk-based prioritization + - Decision documentation + + 3_action: + - Revoke unnecessary access + - Update exceptions + - Document decisions + + 4_report: + - Compliance metrics + - Remediation tracking +``` + +## AWS IAM Review + +```bash +# Generate credential report +aws iam generate-credential-report +aws iam get-credential-report --output text --query Content | base64 -d + +# Find inactive users +aws iam list-users | jq -r '.Users[] | select(.PasswordLastUsed < "2024-01-01") | .UserName' + +# List unused access keys +aws iam get-access-key-last-used --access-key-id AKIAXXXXXXXX +``` + +## Automation + +```python +def generate_access_report(): + users = get_all_users() + report = [] + + for user in users: + report.append({ + 'user': user.email, + 'roles': user.roles, + 'last_login': user.last_login, + 'manager': user.manager, + 'review_status': 'pending' + }) + + return report +``` + +## Best Practices + +- Quarterly reviews minimum +- Risk-based frequency +- Manager attestation +- Automated revocation +- Audit trail maintenance diff --git a/compliance/governance/asset-inventory/SKILL.md b/compliance/governance/asset-inventory/SKILL.md new file mode 100644 index 0000000..823efff --- /dev/null +++ b/compliance/governance/asset-inventory/SKILL.md @@ -0,0 +1,74 @@ +--- +name: asset-inventory +description: Maintain IT asset inventory and configuration management database. Track hardware, software, and cloud resources. Use when managing IT assets. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# Asset Inventory + +Maintain comprehensive IT asset tracking. + +## Asset Categories + +```yaml +asset_types: + hardware: + - Servers + - Network devices + - Endpoints + + software: + - Applications + - Operating systems + - Licenses + + cloud: + - Compute instances + - Storage + - Databases + + data: + - Databases + - File shares + - Backups +``` + +## AWS Inventory + +```bash +# List all resources +aws resourcegroupstaggingapi get-resources + +# EC2 instances +aws ec2 describe-instances --query 'Reservations[*].Instances[*].[InstanceId,InstanceType,State.Name]' + +# AWS Config +aws configservice describe-configuration-recorders +``` + +## Asset Database Schema + +```yaml +asset: + id: unique identifier + name: display name + type: hardware/software/cloud + owner: responsible team + classification: public/internal/confidential + location: physical/cloud location + status: active/retired/decommissioned + created: timestamp + updated: timestamp + tags: [] +``` + +## Best Practices + +- Automated discovery +- Regular reconciliation +- Owner assignment +- Classification tagging +- Lifecycle tracking diff --git a/compliance/governance/change-management/SKILL.md b/compliance/governance/change-management/SKILL.md new file mode 100644 index 0000000..e81b5c9 --- /dev/null +++ b/compliance/governance/change-management/SKILL.md @@ -0,0 +1,81 @@ +--- +name: change-management +description: Implement change management processes. Configure CAB reviews, change windows, and rollback procedures. Use when managing production changes. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# Change Management + +Implement structured change management processes. + +## Change Process + +```yaml +change_workflow: + 1_request: + - Change description + - Risk assessment + - Rollback plan + - Testing evidence + + 2_review: + - Technical review + - Security review + - CAB approval (if high risk) + + 3_schedule: + - Change window + - Communication + - Resource allocation + + 4_implement: + - Execute change + - Verify success + - Update documentation + + 5_review: + - Post-implementation review + - Lessons learned +``` + +## Change Classification + +| Type | Risk | Approval | Example | +|------|------|----------|---------| +| Standard | Low | Pre-approved | Patching | +| Normal | Medium | Manager | Config change | +| Emergency | Variable | Expedited | Security fix | + +## Pull Request Template + +```markdown +## Change Description + +## Risk Level +- [ ] Low - Standard change +- [ ] Medium - Normal change +- [ ] High - CAB required + +## Testing +- [ ] Unit tests pass +- [ ] Integration tests pass +- [ ] Staging deployment verified + +## Rollback Plan + +## Stakeholders Notified +- [ ] Operations +- [ ] Security +- [ ] Business owners +``` + +## Best Practices + +- Clear change categories +- Required approvals by risk +- Rollback procedures documented +- Post-change verification +- Change freeze windows diff --git a/compliance/governance/policy-as-code/SKILL.md b/compliance/governance/policy-as-code/SKILL.md new file mode 100644 index 0000000..ba5f399 --- /dev/null +++ b/compliance/governance/policy-as-code/SKILL.md @@ -0,0 +1,69 @@ +--- +name: policy-as-code +description: Implement policy as code with OPA, Sentinel, and Kyverno. Automate policy enforcement in CI/CD and infrastructure. Use when enforcing compliance through automation. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# Policy as Code + +Automate policy enforcement through code. + +## Open Policy Agent (OPA) + +```rego +# deny_public_buckets.rego +package terraform.s3 + +deny[msg] { + resource := input.resource.aws_s3_bucket[name] + resource.acl == "public-read" + msg := sprintf("S3 bucket '%s' has public ACL", [name]) +} +``` + +## Kyverno (Kubernetes) + +```yaml +apiVersion: kyverno.io/v1 +kind: ClusterPolicy +metadata: + name: require-labels +spec: + validationFailureAction: enforce + rules: + - name: check-labels + match: + resources: + kinds: + - Pod + validate: + message: "Label 'app' is required" + pattern: + metadata: + labels: + app: "?*" +``` + +## Checkov + +```bash +# Scan Terraform +checkov -d . --framework terraform + +# Custom check +from checkov.terraform.checks.resource.base_resource_check import BaseResourceCheck + +class S3Encryption(BaseResourceCheck): + def scan_resource_conf(self, conf): + return CheckResult.PASSED if 'encryption' in conf else CheckResult.FAILED +``` + +## Best Practices + +- Version control policies +- Test policies in CI +- Gradual rollout (warn → enforce) +- Exception management diff --git a/compliance/governance/vendor-management/SKILL.md b/compliance/governance/vendor-management/SKILL.md new file mode 100644 index 0000000..859a02e --- /dev/null +++ b/compliance/governance/vendor-management/SKILL.md @@ -0,0 +1,73 @@ +--- +name: vendor-management +description: Implement vendor risk management programs. Assess third-party security and maintain vendor inventory. Use when managing supplier security. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# Vendor Management + +Manage third-party vendor security risks. + +## Vendor Assessment + +```yaml +assessment_process: + 1_identify: + - Catalog all vendors + - Classify by risk tier + + 2_assess: + - Security questionnaire + - SOC 2 review + - Penetration test results + + 3_contract: + - Security requirements + - Data processing agreement + - SLAs + + 4_monitor: + - Continuous monitoring + - Annual reassessment + - Incident notification +``` + +## Risk Tiers + +| Tier | Criteria | Assessment | +|------|----------|------------| +| Critical | Access to sensitive data | Full assessment, annual | +| High | Significant data access | Questionnaire + SOC 2 | +| Medium | Limited data access | Security questionnaire | +| Low | No data access | Basic due diligence | + +## Security Questionnaire + +```yaml +categories: + governance: + - Security policies + - Risk management + - Compliance certifications + + technical: + - Access controls + - Encryption + - Vulnerability management + + operational: + - Incident response + - Business continuity + - Change management +``` + +## Best Practices + +- Tier-based assessments +- Regular reassessment +- Contract security terms +- Incident notification requirements +- Exit strategy planning diff --git a/devops/ci-cd/azure-devops/SKILL.md b/devops/ci-cd/azure-devops/SKILL.md new file mode 100644 index 0000000..5a0f671 --- /dev/null +++ b/devops/ci-cd/azure-devops/SKILL.md @@ -0,0 +1,432 @@ +--- +name: azure-devops +description: Set up Azure Pipelines for CI/CD, configure build and release pipelines, manage Azure DevOps projects, and integrate with Azure services. Use when working with Azure DevOps Services or Server for enterprise DevOps workflows. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# Azure DevOps Pipelines + +Build, test, and deploy applications using Azure Pipelines with YAML or classic editor. + +## When to Use This Skill + +Use this skill when: +- Creating CI/CD pipelines in Azure DevOps +- Configuring build and release stages +- Managing Azure DevOps service connections +- Deploying to Azure or other cloud platforms +- Setting up multi-stage YAML pipelines + +## Prerequisites + +- Azure DevOps organization and project +- Service connections for target environments +- Basic YAML understanding +- Azure subscription (for Azure deployments) + +## YAML Pipeline Structure + +Create `azure-pipelines.yml` in repository root: + +```yaml +trigger: + branches: + include: + - main + - develop + paths: + include: + - src/* + +pool: + vmImage: 'ubuntu-latest' + +variables: + buildConfiguration: 'Release' + nodeVersion: '20.x' + +stages: + - stage: Build + jobs: + - job: BuildJob + steps: + - task: NodeTool@0 + inputs: + versionSpec: $(nodeVersion) + - script: | + npm ci + npm run build + displayName: 'Build application' + - publish: $(Build.ArtifactStagingDirectory) + artifact: drop + + - stage: Deploy + dependsOn: Build + condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/main')) + jobs: + - deployment: DeployWeb + environment: 'production' + strategy: + runOnce: + deploy: + steps: + - script: echo Deploying to production +``` + +## Triggers + +### Branch Triggers + +```yaml +trigger: + branches: + include: + - main + - release/* + exclude: + - feature/* + tags: + include: + - v* +``` + +### Pull Request Triggers + +```yaml +pr: + branches: + include: + - main + paths: + include: + - src/* + exclude: + - docs/* +``` + +### Scheduled Triggers + +```yaml +schedules: + - cron: '0 2 * * *' + displayName: 'Nightly build' + branches: + include: + - main + always: true +``` + +## Jobs and Stages + +### Parallel Jobs + +```yaml +stages: + - stage: Test + jobs: + - job: UnitTests + pool: + vmImage: 'ubuntu-latest' + steps: + - script: npm run test:unit + + - job: IntegrationTests + pool: + vmImage: 'ubuntu-latest' + steps: + - script: npm run test:integration +``` + +### Matrix Strategy + +```yaml +jobs: + - job: Build + strategy: + matrix: + linux: + vmImage: 'ubuntu-latest' + windows: + vmImage: 'windows-latest' + mac: + vmImage: 'macos-latest' + pool: + vmImage: $(vmImage) + steps: + - script: npm test +``` + +### Job Dependencies + +```yaml +stages: + - stage: Build + jobs: + - job: A + steps: + - script: echo Job A + - job: B + dependsOn: A + steps: + - script: echo Job B +``` + +## Variables and Parameters + +### Variable Groups + +```yaml +variables: + - group: 'production-secrets' + - name: buildConfiguration + value: 'Release' +``` + +### Runtime Parameters + +```yaml +parameters: + - name: environment + displayName: 'Environment' + type: string + default: 'dev' + values: + - dev + - staging + - prod + +stages: + - stage: Deploy + variables: + env: ${{ parameters.environment }} + jobs: + - job: Deploy + steps: + - script: echo "Deploying to $(env)" +``` + +### Secret Variables + +```yaml +variables: + - name: mySecret + value: $(SECRET_FROM_PIPELINE) # Set in pipeline settings + +steps: + - script: | + echo "Using secret" + ./deploy.sh + env: + API_KEY: $(mySecret) +``` + +## Templates + +### Job Template + +```yaml +# templates/build-job.yml +parameters: + - name: nodeVersion + default: '20' + +jobs: + - job: Build + steps: + - task: NodeTool@0 + inputs: + versionSpec: ${{ parameters.nodeVersion }} + - script: npm ci && npm run build +``` + +### Using Templates + +```yaml +# azure-pipelines.yml +stages: + - stage: Build + jobs: + - template: templates/build-job.yml + parameters: + nodeVersion: '20' +``` + +### Stage Template + +```yaml +# templates/deploy-stage.yml +parameters: + - name: environment + type: string + - name: serviceConnection + type: string + +stages: + - stage: Deploy_${{ parameters.environment }} + jobs: + - deployment: Deploy + environment: ${{ parameters.environment }} + strategy: + runOnce: + deploy: + steps: + - task: AzureWebApp@1 + inputs: + azureSubscription: ${{ parameters.serviceConnection }} + appName: 'myapp-${{ parameters.environment }}' +``` + +## Deployments + +### Environment Deployments + +```yaml +stages: + - stage: DeployStaging + jobs: + - deployment: DeployWeb + environment: 'staging' + strategy: + runOnce: + deploy: + steps: + - download: current + artifact: drop + - script: ./deploy.sh staging +``` + +### Approval Gates + +Configure in Azure DevOps UI: +1. Go to Environments +2. Select environment +3. Add approval check +4. Configure approvers + +### Rolling Deployment + +```yaml +jobs: + - deployment: Deploy + environment: 'production' + strategy: + rolling: + maxParallel: 2 + deploy: + steps: + - script: ./deploy.sh +``` + +## Azure Service Tasks + +### Azure Web App Deployment + +```yaml +- task: AzureWebApp@1 + inputs: + azureSubscription: 'my-azure-connection' + appType: 'webAppLinux' + appName: 'my-web-app' + package: '$(Pipeline.Workspace)/drop/*.zip' +``` + +### Azure Container Apps + +```yaml +- task: AzureContainerApps@1 + inputs: + azureSubscription: 'my-azure-connection' + containerAppName: 'my-container-app' + resourceGroup: 'my-rg' + imageToDeploy: 'myregistry.azurecr.io/myapp:$(Build.BuildId)' +``` + +### Azure Kubernetes Service + +```yaml +- task: KubernetesManifest@0 + inputs: + action: 'deploy' + kubernetesServiceConnection: 'my-aks-connection' + namespace: 'default' + manifests: | + $(Pipeline.Workspace)/manifests/deployment.yml + $(Pipeline.Workspace)/manifests/service.yml + containers: | + myregistry.azurecr.io/myapp:$(Build.BuildId) +``` + +## Docker Builds + +```yaml +- task: Docker@2 + inputs: + containerRegistry: 'my-acr-connection' + repository: 'myapp' + command: 'buildAndPush' + Dockerfile: '**/Dockerfile' + tags: | + $(Build.BuildId) + latest +``` + +## Self-Hosted Agents + +### Install Agent + +```bash +# Download agent +mkdir myagent && cd myagent +curl -o vsts-agent.tar.gz https://vstsagentpackage.azureedge.net/agent/3.227.2/vsts-agent-linux-x64-3.227.2.tar.gz +tar zxvf vsts-agent.tar.gz + +# Configure +./config.sh --url https://dev.azure.com/myorg --auth pat --token PAT_TOKEN --pool default + +# Run as service +sudo ./svc.sh install +sudo ./svc.sh start +``` + +### Use Self-Hosted Pool + +```yaml +pool: + name: 'my-self-hosted-pool' + demands: + - docker + - Agent.OS -equals Linux +``` + +## Common Issues + +### Issue: Service Connection Fails +**Problem**: Cannot authenticate to Azure +**Solution**: Verify service principal permissions, check connection in project settings + +### Issue: Artifact Not Found +**Problem**: Download artifact fails +**Solution**: Ensure publish task ran successfully, check artifact name matches + +### Issue: Environment Not Found +**Problem**: Deployment to environment fails +**Solution**: Create environment in Pipelines > Environments first + +## Best Practices + +- Use YAML pipelines over classic editor +- Implement templates for reusable components +- Use variable groups for shared configuration +- Configure environment approvals for production +- Use service connections with minimal permissions +- Implement artifact versioning +- Cache dependencies for faster builds + +## Related Skills + +- [github-actions](../github-actions/) - GitHub CI/CD alternative +- [terraform-azure](../../../infrastructure/cloud-azure/terraform-azure/) - Azure IaC +- [azure-aks](../../../infrastructure/cloud-azure/azure-aks/) - AKS deployments diff --git a/devops/ci-cd/circleci/SKILL.md b/devops/ci-cd/circleci/SKILL.md new file mode 100644 index 0000000..358bd6f --- /dev/null +++ b/devops/ci-cd/circleci/SKILL.md @@ -0,0 +1,450 @@ +--- +name: circleci +description: Configure CircleCI workflows and orbs for continuous integration and deployment. Create config.yml pipelines, use orbs for reusable configurations, and optimize build performance. Use when working with CircleCI for CI/CD automation. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# CircleCI + +Build, test, and deploy applications using CircleCI's cloud-native CI/CD platform. + +## When to Use This Skill + +Use this skill when: +- Setting up CI/CD pipelines with CircleCI +- Using orbs for reusable configuration +- Optimizing build times with caching and parallelism +- Configuring CircleCI workflows and approvals +- Managing CircleCI contexts and secrets + +## Prerequisites + +- CircleCI account connected to repository +- Project enabled in CircleCI dashboard +- Basic YAML understanding + +## Configuration File + +Create `.circleci/config.yml`: + +```yaml +version: 2.1 + +orbs: + node: circleci/node@5.2 + docker: circleci/docker@2.4 + +executors: + default: + docker: + - image: cimg/node:20.10 + working_directory: ~/project + +jobs: + build: + executor: default + steps: + - checkout + - node/install-packages: + pkg-manager: npm + - run: + name: Build application + command: npm run build + - persist_to_workspace: + root: . + paths: + - dist + + test: + executor: default + steps: + - checkout + - node/install-packages: + pkg-manager: npm + - run: + name: Run tests + command: npm test + + deploy: + executor: default + steps: + - checkout + - attach_workspace: + at: . + - run: + name: Deploy + command: ./deploy.sh + +workflows: + build-test-deploy: + jobs: + - build + - test: + requires: + - build + - deploy: + requires: + - test + filters: + branches: + only: main +``` + +## Executors + +### Docker Executor + +```yaml +executors: + node: + docker: + - image: cimg/node:20.10 + - image: cimg/postgres:15.0 + environment: + POSTGRES_USER: test + POSTGRES_DB: testdb + working_directory: ~/app +``` + +### Machine Executor + +```yaml +executors: + linux-machine: + machine: + image: ubuntu-2204:current + resource_class: large +``` + +### macOS Executor + +```yaml +executors: + macos: + macos: + xcode: "15.0.0" + resource_class: macos.m1.medium.gen1 +``` + +## Caching + +### Dependency Caching + +```yaml +jobs: + build: + steps: + - checkout + - restore_cache: + keys: + - v1-deps-{{ checksum "package-lock.json" }} + - v1-deps- + - run: npm ci + - save_cache: + key: v1-deps-{{ checksum "package-lock.json" }} + paths: + - node_modules +``` + +### Multi-Key Caching + +```yaml +- restore_cache: + keys: + - v1-{{ .Branch }}-{{ checksum "package-lock.json" }} + - v1-{{ .Branch }}- + - v1-main- + - v1- +``` + +## Workspaces + +### Persist Data + +```yaml +jobs: + build: + steps: + - checkout + - run: npm run build + - persist_to_workspace: + root: . + paths: + - dist + - node_modules + + deploy: + steps: + - attach_workspace: + at: ~/project + - run: ./deploy.sh +``` + +## Parallelism + +### Test Splitting + +```yaml +jobs: + test: + parallelism: 4 + steps: + - checkout + - run: + name: Run tests + command: | + TESTFILES=$(circleci tests glob "test/**/*.test.js" | circleci tests split --split-by=timings) + npm test -- $TESTFILES + - store_test_results: + path: test-results +``` + +## Workflows + +### Sequential Jobs + +```yaml +workflows: + pipeline: + jobs: + - build + - test: + requires: + - build + - deploy: + requires: + - test +``` + +### Parallel Jobs + +```yaml +workflows: + pipeline: + jobs: + - build + - test-unit: + requires: + - build + - test-integration: + requires: + - build + - deploy: + requires: + - test-unit + - test-integration +``` + +### Manual Approval + +```yaml +workflows: + deploy-prod: + jobs: + - build + - test + - hold: + type: approval + requires: + - test + - deploy-production: + requires: + - hold +``` + +### Scheduled Workflows + +```yaml +workflows: + nightly: + triggers: + - schedule: + cron: "0 2 * * *" + filters: + branches: + only: + - main + jobs: + - build + - test +``` + +### Branch Filtering + +```yaml +workflows: + build-deploy: + jobs: + - build: + filters: + branches: + only: + - main + - /feature-.*/ + - deploy: + filters: + branches: + only: main + tags: + only: /^v.*/ +``` + +## Orbs + +### Using Orbs + +```yaml +version: 2.1 + +orbs: + aws-cli: circleci/aws-cli@4.1 + kubernetes: circleci/kubernetes@1.3 + +jobs: + deploy: + executor: aws-cli/default + steps: + - aws-cli/setup: + aws_access_key_id: AWS_ACCESS_KEY_ID + aws_secret_access_key: AWS_SECRET_ACCESS_KEY + - kubernetes/install-kubectl + - run: kubectl apply -f k8s/ +``` + +### Common Orbs + +```yaml +orbs: + node: circleci/node@5.2 # Node.js + docker: circleci/docker@2.4 # Docker builds + aws-cli: circleci/aws-cli@4.1 # AWS CLI + aws-ecr: circleci/aws-ecr@9.0 # ECR push + aws-ecs: circleci/aws-ecs@4.0 # ECS deploy + gcp-cli: circleci/gcp-cli@3.1 # GCP CLI + kubernetes: circleci/kubernetes@1.3 # K8s deploy + slack: circleci/slack@4.12 # Notifications +``` + +## Docker Builds + +```yaml +version: 2.1 + +orbs: + docker: circleci/docker@2.4 + +jobs: + build-and-push: + executor: docker/docker + steps: + - setup_remote_docker: + version: 20.10.24 + - checkout + - docker/check + - docker/build: + image: myorg/myapp + tag: $CIRCLE_SHA1 + - docker/push: + image: myorg/myapp + tag: $CIRCLE_SHA1 +``` + +## Environment Variables + +### Project Variables + +Set in CircleCI Project Settings > Environment Variables + +### Contexts + +```yaml +workflows: + deploy: + jobs: + - deploy-staging: + context: staging-secrets + - deploy-production: + context: production-secrets +``` + +### Using Variables + +```yaml +jobs: + deploy: + steps: + - run: + name: Deploy + command: | + aws s3 sync dist/ s3://$S3_BUCKET + environment: + AWS_DEFAULT_REGION: us-east-1 +``` + +## Artifacts and Test Results + +```yaml +jobs: + test: + steps: + - run: + name: Run tests + command: npm test -- --coverage + - store_test_results: + path: test-results + - store_artifacts: + path: coverage + destination: coverage-report +``` + +## Resource Classes + +```yaml +jobs: + build: + docker: + - image: cimg/node:20.10 + resource_class: large # 4 vCPU, 8GB RAM + steps: + - checkout + - run: npm run build + +# Available classes: +# small: 1 vCPU, 2GB RAM +# medium: 2 vCPU, 4GB RAM (default) +# large: 4 vCPU, 8GB RAM +# xlarge: 8 vCPU, 16GB RAM +``` + +## Common Issues + +### Issue: Cache Not Restoring +**Problem**: Cache misses on every build +**Solution**: Verify cache key format, ensure checksum file hasn't changed + +### Issue: Workspace Attach Fails +**Problem**: Cannot find persisted workspace +**Solution**: Ensure persist_to_workspace job completed, check paths + +### Issue: Docker Layer Caching +**Problem**: Docker builds are slow +**Solution**: Enable Docker Layer Caching in project settings (paid feature) + +## Best Practices + +- Use orbs for common tasks +- Implement aggressive caching strategies +- Use workspaces for sharing data between jobs +- Split tests with parallelism for faster builds +- Use contexts for environment-specific secrets +- Define reusable executors +- Store test results for insights + +## Related Skills + +- [github-actions](../github-actions/) - GitHub CI/CD +- [docker-management](../../containers/docker-management/) - Container builds +- [aws-ecs-fargate](../../../infrastructure/cloud-aws/aws-ecs-fargate/) - ECS deployments diff --git a/devops/ci-cd/github-actions/SKILL.md b/devops/ci-cd/github-actions/SKILL.md new file mode 100644 index 0000000..3dbe453 --- /dev/null +++ b/devops/ci-cd/github-actions/SKILL.md @@ -0,0 +1,344 @@ +--- +name: github-actions +description: Build, test, and deploy applications using GitHub Actions workflows. Create CI/CD pipelines, configure runners, manage secrets, and automate software delivery. Use when working with GitHub repositories, automating builds, running tests, or deploying applications. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# GitHub Actions + +Automate software workflows directly in your GitHub repository with GitHub Actions. + +## When to Use This Skill + +Use this skill when: +- Setting up CI/CD pipelines for GitHub repositories +- Automating build, test, and deployment workflows +- Creating reusable workflow components +- Configuring self-hosted runners +- Managing workflow secrets and variables +- Debugging failed workflow runs + +## Prerequisites + +- GitHub repository with write access +- Understanding of YAML syntax +- For self-hosted runners: server with Docker (optional) + +## Workflow File Structure + +Workflows are defined in `.github/workflows/` directory: + +```yaml +name: CI Pipeline + +on: + push: + branches: [main, develop] + pull_request: + branches: [main] + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + cache: 'npm' + - run: npm ci + - run: npm test +``` + +## Common Triggers + +### Push and Pull Request + +```yaml +on: + push: + branches: [main] + paths: + - 'src/**' + - 'package.json' + pull_request: + branches: [main] +``` + +### Scheduled Runs + +```yaml +on: + schedule: + - cron: '0 2 * * *' # Daily at 2 AM UTC +``` + +### Manual Dispatch + +```yaml +on: + workflow_dispatch: + inputs: + environment: + description: 'Deployment environment' + required: true + default: 'staging' + type: choice + options: + - staging + - production +``` + +## Job Configuration + +### Matrix Builds + +```yaml +jobs: + test: + runs-on: ubuntu-latest + strategy: + matrix: + node-version: [18, 20, 22] + os: [ubuntu-latest, windows-latest] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node-version }} + - run: npm test +``` + +### Job Dependencies + +```yaml +jobs: + build: + runs-on: ubuntu-latest + steps: + - run: npm run build + + test: + needs: build + runs-on: ubuntu-latest + steps: + - run: npm test + + deploy: + needs: [build, test] + runs-on: ubuntu-latest + steps: + - run: ./deploy.sh +``` + +### Environment Protection + +```yaml +jobs: + deploy: + runs-on: ubuntu-latest + environment: + name: production + url: https://example.com + steps: + - run: ./deploy.sh +``` + +## Secrets and Variables + +### Using Secrets + +```yaml +steps: + - name: Deploy + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + run: aws s3 sync ./dist s3://my-bucket +``` + +### Using Variables + +```yaml +steps: + - name: Build + env: + API_URL: ${{ vars.API_URL }} + run: npm run build +``` + +## Caching Dependencies + +```yaml +- uses: actions/cache@v4 + with: + path: ~/.npm + key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }} + restore-keys: | + ${{ runner.os }}-node- +``` + +## Artifacts + +### Upload Artifacts + +```yaml +- uses: actions/upload-artifact@v4 + with: + name: build-output + path: dist/ + retention-days: 5 +``` + +### Download Artifacts + +```yaml +- uses: actions/download-artifact@v4 + with: + name: build-output + path: dist/ +``` + +## Docker Builds + +```yaml +jobs: + docker: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Build and push + uses: docker/build-push-action@v5 + with: + context: . + push: true + tags: user/app:latest +``` + +## Reusable Workflows + +### Define Reusable Workflow + +```yaml +# .github/workflows/reusable-deploy.yml +name: Reusable Deploy + +on: + workflow_call: + inputs: + environment: + required: true + type: string + secrets: + deploy_key: + required: true + +jobs: + deploy: + runs-on: ubuntu-latest + environment: ${{ inputs.environment }} + steps: + - run: echo "Deploying to ${{ inputs.environment }}" +``` + +### Call Reusable Workflow + +```yaml +jobs: + deploy-staging: + uses: ./.github/workflows/reusable-deploy.yml + with: + environment: staging + secrets: + deploy_key: ${{ secrets.STAGING_KEY }} +``` + +## Self-Hosted Runners + +### Register Runner + +```bash +# Download runner +mkdir actions-runner && cd actions-runner +curl -o actions-runner-linux-x64.tar.gz -L https://github.com/actions/runner/releases/download/v2.311.0/actions-runner-linux-x64-2.311.0.tar.gz +tar xzf actions-runner-linux-x64.tar.gz + +# Configure +./config.sh --url https://github.com/OWNER/REPO --token TOKEN + +# Run +./run.sh +``` + +### Use Self-Hosted Runner + +```yaml +jobs: + build: + runs-on: self-hosted + steps: + - uses: actions/checkout@v4 +``` + +## Debugging Workflows + +### Enable Debug Logging + +Set repository secrets: +- `ACTIONS_RUNNER_DEBUG`: `true` +- `ACTIONS_STEP_DEBUG`: `true` + +### Debug Step + +```yaml +- name: Debug + run: | + echo "GitHub context: ${{ toJson(github) }}" + echo "Job context: ${{ toJson(job) }}" +``` + +## Common Issues + +### Issue: Workflow Not Triggering +**Problem**: Workflow doesn't run on push/PR +**Solution**: Check branch filters, path filters, and ensure workflow file is on the default branch + +### Issue: Permission Denied +**Problem**: Actions can't push or create PRs +**Solution**: Configure `permissions` in workflow or update repository settings + +```yaml +permissions: + contents: write + pull-requests: write +``` + +### Issue: Cache Not Restoring +**Problem**: Cache misses despite existing cache +**Solution**: Verify cache key matches exactly, check runner OS + +## Best Practices + +- Pin action versions to specific commits or tags +- Use caching for dependencies to speed up builds +- Minimize secrets exposure with environment scoping +- Use matrix builds for cross-platform testing +- Implement proper error handling with `continue-on-error` +- Keep workflows DRY with reusable workflows and composite actions + +## Related Skills + +- [gitlab-ci](../gitlab-ci/) - GitLab CI/CD alternative +- [docker-management](../../containers/docker-management/) - Container builds +- [semantic-versioning](../../release/semantic-versioning/) - Automated releases diff --git a/devops/ci-cd/github-actions/references/workflow-patterns.md b/devops/ci-cd/github-actions/references/workflow-patterns.md new file mode 100644 index 0000000..bb2577a --- /dev/null +++ b/devops/ci-cd/github-actions/references/workflow-patterns.md @@ -0,0 +1,125 @@ +# GitHub Actions Workflow Patterns + +## Reusable Workflows + +### Caller Workflow +```yaml +jobs: + call-workflow: + uses: org/repo/.github/workflows/reusable.yml@main + with: + environment: production + secrets: + deploy_key: ${{ secrets.DEPLOY_KEY }} +``` + +### Reusable Workflow +```yaml +# .github/workflows/reusable.yml +name: Reusable Deploy + +on: + workflow_call: + inputs: + environment: + required: true + type: string + secrets: + deploy_key: + required: true + +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - run: ./deploy.sh ${{ inputs.environment }} +``` + +## Matrix Builds + +```yaml +strategy: + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + node: [18, 20] + exclude: + - os: macos-latest + node: 18 + include: + - os: ubuntu-latest + node: 20 + experimental: true + fail-fast: false + +steps: + - uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node }} +``` + +## Environment Protection + +```yaml +jobs: + deploy: + runs-on: ubuntu-latest + environment: + name: production + url: https://example.com + steps: + - run: ./deploy.sh +``` + +## Concurrency Control + +```yaml +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true +``` + +## Job Dependencies + +```yaml +jobs: + build: + runs-on: ubuntu-latest + steps: [...] + + test: + needs: build + runs-on: ubuntu-latest + steps: [...] + + deploy: + needs: [build, test] + if: github.ref == 'refs/heads/main' + runs-on: ubuntu-latest +``` + +## Artifact Sharing + +```yaml +- uses: actions/upload-artifact@v4 + with: + name: build-output + path: dist/ + retention-days: 5 + +- uses: actions/download-artifact@v4 + with: + name: build-output + path: dist/ +``` + +## Caching + +```yaml +- uses: actions/cache@v4 + with: + path: ~/.npm + key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }} + restore-keys: | + ${{ runner.os }}-node- +``` diff --git a/devops/ci-cd/gitlab-ci/SKILL.md b/devops/ci-cd/gitlab-ci/SKILL.md new file mode 100644 index 0000000..c3d41b9 --- /dev/null +++ b/devops/ci-cd/gitlab-ci/SKILL.md @@ -0,0 +1,315 @@ +--- +name: gitlab-ci +description: Configure GitLab CI/CD pipelines and runners for automated building, testing, and deployment. Create .gitlab-ci.yml configurations, manage runners, and implement DevOps workflows. Use when working with GitLab repositories or self-hosted GitLab instances. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# GitLab CI/CD + +Automate your software delivery pipeline with GitLab's integrated CI/CD system. + +## When to Use This Skill + +Use this skill when: +- Setting up CI/CD pipelines in GitLab +- Configuring GitLab runners (shared or self-hosted) +- Creating multi-stage deployment pipelines +- Implementing GitLab Auto DevOps +- Managing CI/CD variables and secrets + +## Prerequisites + +- GitLab repository (gitlab.com or self-hosted) +- Basic understanding of YAML +- For self-hosted runners: Linux server or Kubernetes cluster + +## Pipeline Configuration + +Create `.gitlab-ci.yml` in repository root: + +```yaml +stages: + - build + - test + - deploy + +variables: + NODE_VERSION: "20" + +build: + stage: build + image: node:${NODE_VERSION} + script: + - npm ci + - npm run build + artifacts: + paths: + - dist/ + expire_in: 1 hour + +test: + stage: test + image: node:${NODE_VERSION} + script: + - npm ci + - npm test + coverage: '/Coverage: \d+\.\d+%/' + +deploy: + stage: deploy + script: + - ./deploy.sh + environment: + name: production + url: https://example.com + only: + - main +``` + +## Job Configuration + +### Rules-Based Execution + +```yaml +deploy: + script: ./deploy.sh + rules: + - if: $CI_COMMIT_BRANCH == "main" + when: manual + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + when: never + - when: on_success +``` + +### Parallel Jobs + +```yaml +test: + stage: test + parallel: 3 + script: + - npm test -- --shard=$CI_NODE_INDEX/$CI_NODE_TOTAL +``` + +### Matrix Builds + +```yaml +test: + stage: test + parallel: + matrix: + - NODE_VERSION: ["18", "20", "22"] + OS: ["alpine", "slim"] + image: node:${NODE_VERSION}-${OS} + script: + - npm test +``` + +## Caching + +```yaml +cache: + key: + files: + - package-lock.json + paths: + - node_modules/ + policy: pull-push + +build: + cache: + key: build-cache + paths: + - .cache/ + policy: pull +``` + +## Artifacts + +```yaml +build: + artifacts: + paths: + - dist/ + - coverage/ + reports: + junit: junit.xml + coverage_report: + coverage_format: cobertura + path: coverage/cobertura.xml + expire_in: 1 week + when: always +``` + +## Environments and Deployments + +```yaml +deploy_staging: + stage: deploy + script: + - deploy --env staging + environment: + name: staging + url: https://staging.example.com + on_stop: stop_staging + +stop_staging: + stage: deploy + script: + - undeploy --env staging + environment: + name: staging + action: stop + when: manual +``` + +## Docker Builds + +```yaml +build_image: + stage: build + image: docker:24 + services: + - docker:24-dind + variables: + DOCKER_TLS_CERTDIR: "/certs" + script: + - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY + - docker build -t $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA . + - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA +``` + +## GitLab Runners + +### Install Runner + +```bash +# Download and install +curl -L https://packages.gitlab.com/install/repositories/runner/gitlab-runner/script.deb.sh | sudo bash +sudo apt install gitlab-runner + +# Register runner +sudo gitlab-runner register \ + --url https://gitlab.com/ \ + --registration-token TOKEN \ + --executor docker \ + --docker-image alpine:latest +``` + +### Runner Configuration + +```toml +# /etc/gitlab-runner/config.toml +[[runners]] + name = "docker-runner" + url = "https://gitlab.com/" + token = "TOKEN" + executor = "docker" + [runners.docker] + image = "alpine:latest" + privileged = true + volumes = ["/cache", "/var/run/docker.sock:/var/run/docker.sock"] +``` + +### Runner Tags + +```yaml +build: + tags: + - docker + - linux + script: + - make build +``` + +## CI/CD Variables + +### Protected Variables + +Define in Settings > CI/CD > Variables: +- `AWS_ACCESS_KEY_ID` (protected, masked) +- `AWS_SECRET_ACCESS_KEY` (protected, masked) + +### Using Variables + +```yaml +deploy: + script: + - aws s3 sync dist/ s3://$S3_BUCKET + variables: + AWS_DEFAULT_REGION: us-east-1 +``` + +## Include and Extend + +### Include Templates + +```yaml +include: + - template: Security/SAST.gitlab-ci.yml + - project: 'group/shared-ci' + file: '/templates/deploy.yml' + - local: '/ci/jobs.yml' +``` + +### Extend Jobs + +```yaml +.base_job: + image: node:20 + before_script: + - npm ci + +build: + extends: .base_job + script: + - npm run build + +test: + extends: .base_job + script: + - npm test +``` + +## Multi-Project Pipelines + +```yaml +trigger_downstream: + stage: deploy + trigger: + project: group/downstream-project + branch: main + strategy: depend +``` + +## Common Issues + +### Issue: Pipeline Stuck +**Problem**: Jobs stay pending +**Solution**: Check runner availability and tags matching + +### Issue: Docker-in-Docker Fails +**Problem**: Cannot connect to Docker daemon +**Solution**: Use `docker:dind` service with proper TLS configuration + +### Issue: Cache Not Working +**Problem**: Cache misses between jobs +**Solution**: Verify cache key and ensure runners share distributed cache + +## Best Practices + +- Use `rules` instead of `only/except` for complex conditions +- Leverage GitLab's built-in security scanning templates +- Use job dependencies to optimize pipeline speed +- Implement review apps for merge requests +- Cache dependencies aggressively +- Use artifacts for passing data between stages + +## Related Skills + +- [github-actions](../github-actions/) - GitHub CI/CD alternative +- [argocd-gitops](../../orchestration/argocd-gitops/) - GitOps deployments +- [container-registries](../../containers/container-registries/) - Registry management diff --git a/devops/ci-cd/gitlab-ci/references/pipeline-patterns.md b/devops/ci-cd/gitlab-ci/references/pipeline-patterns.md new file mode 100644 index 0000000..ada7436 --- /dev/null +++ b/devops/ci-cd/gitlab-ci/references/pipeline-patterns.md @@ -0,0 +1,113 @@ +# GitLab CI Pipeline Patterns + +## Parent-Child Pipelines + +```yaml +# Parent pipeline +stages: + - triggers + +trigger-services: + stage: triggers + trigger: + include: + - local: services/api/.gitlab-ci.yml + - local: services/web/.gitlab-ci.yml + strategy: depend +``` + +## DAG (Directed Acyclic Graph) + +```yaml +build: + stage: build + script: make build + +test-unit: + stage: test + needs: [build] + script: make test-unit + +test-integration: + stage: test + needs: [build] + script: make test-integration + +deploy: + stage: deploy + needs: [test-unit, test-integration] + script: make deploy +``` + +## Dynamic Child Pipelines + +```yaml +generate-config: + stage: prepare + script: + - generate-pipeline.sh > child-pipeline.yml + artifacts: + paths: + - child-pipeline.yml + +trigger-child: + stage: trigger + trigger: + include: + - artifact: child-pipeline.yml + job: generate-config +``` + +## Multi-Project Pipelines + +```yaml +deploy-downstream: + trigger: + project: group/downstream-project + branch: main + strategy: depend + variables: + UPSTREAM_VERSION: $CI_COMMIT_SHA +``` + +## Rules and Conditions + +```yaml +deploy: + rules: + - if: $CI_COMMIT_BRANCH == "main" + when: manual + - if: $CI_COMMIT_TAG + when: on_success + - when: never +``` + +## Caching Strategies + +```yaml +default: + cache: + key: + files: + - package-lock.json + paths: + - node_modules/ + policy: pull-push + +test: + cache: + policy: pull # Only read from cache +``` + +## Services + +```yaml +test: + services: + - name: postgres:15 + alias: db + - name: redis:7 + variables: + POSTGRES_DB: test + DATABASE_URL: postgres://postgres@db/test +``` diff --git a/devops/ci-cd/jenkins/SKILL.md b/devops/ci-cd/jenkins/SKILL.md new file mode 100644 index 0000000..e36d169 --- /dev/null +++ b/devops/ci-cd/jenkins/SKILL.md @@ -0,0 +1,437 @@ +--- +name: jenkins +description: Create and manage Jenkins CI/CD pipelines, configure agents, manage plugins, and automate builds. Use when working with Jenkins servers, creating Jenkinsfiles, or setting up build automation for enterprise environments. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# Jenkins + +Build, test, and deploy applications using Jenkins, the leading open-source automation server. + +## When to Use This Skill + +Use this skill when: +- Setting up Jenkins pipelines (declarative or scripted) +- Configuring Jenkins agents and executors +- Managing Jenkins plugins and security +- Creating shared libraries for pipeline reuse +- Integrating Jenkins with external tools + +## Prerequisites + +- Jenkins server (2.x or later) +- Admin access to Jenkins +- Java 11+ on Jenkins server +- Basic Groovy understanding for pipelines + +## Declarative Pipeline + +Create `Jenkinsfile` in repository root: + +```groovy +pipeline { + agent any + + environment { + DOCKER_REGISTRY = 'registry.example.com' + APP_NAME = 'myapp' + } + + stages { + stage('Build') { + steps { + sh 'npm ci' + sh 'npm run build' + } + } + + stage('Test') { + steps { + sh 'npm test' + } + post { + always { + junit 'test-results/*.xml' + } + } + } + + stage('Deploy') { + when { + branch 'main' + } + steps { + sh './deploy.sh' + } + } + } + + post { + failure { + mail to: 'team@example.com', + subject: "Pipeline Failed: ${env.JOB_NAME}", + body: "Check console output at ${env.BUILD_URL}" + } + } +} +``` + +## Agent Configuration + +### Docker Agent + +```groovy +pipeline { + agent { + docker { + image 'node:20' + args '-v /tmp:/tmp' + } + } + stages { + stage('Build') { + steps { + sh 'npm ci && npm run build' + } + } + } +} +``` + +### Kubernetes Agent + +```groovy +pipeline { + agent { + kubernetes { + yaml ''' + apiVersion: v1 + kind: Pod + spec: + containers: + - name: node + image: node:20 + command: + - sleep + args: + - infinity + - name: docker + image: docker:24-dind + securityContext: + privileged: true + ''' + } + } + stages { + stage('Build') { + steps { + container('node') { + sh 'npm ci && npm run build' + } + } + } + } +} +``` + +### Labeled Agents + +```groovy +pipeline { + agent { label 'linux && docker' } + stages { + stage('Build') { + steps { + sh 'make build' + } + } + } +} +``` + +## Parameters + +```groovy +pipeline { + agent any + + parameters { + string(name: 'BRANCH', defaultValue: 'main', description: 'Branch to build') + choice(name: 'ENVIRONMENT', choices: ['dev', 'staging', 'prod'], description: 'Target environment') + booleanParam(name: 'RUN_TESTS', defaultValue: true, description: 'Run tests?') + } + + stages { + stage('Deploy') { + when { + expression { params.ENVIRONMENT == 'prod' } + } + steps { + sh "deploy.sh ${params.ENVIRONMENT}" + } + } + } +} +``` + +## Credentials + +### Using Credentials + +```groovy +pipeline { + agent any + + environment { + AWS_CREDS = credentials('aws-credentials') + DOCKER_CREDS = credentials('docker-hub') + } + + stages { + stage('Deploy') { + steps { + withCredentials([ + usernamePassword( + credentialsId: 'github-token', + usernameVariable: 'GH_USER', + passwordVariable: 'GH_TOKEN' + ) + ]) { + sh 'git push https://${GH_USER}:${GH_TOKEN}@github.com/repo.git' + } + } + } + } +} +``` + +## Parallel Stages + +```groovy +pipeline { + agent any + + stages { + stage('Tests') { + parallel { + stage('Unit Tests') { + steps { + sh 'npm run test:unit' + } + } + stage('Integration Tests') { + steps { + sh 'npm run test:integration' + } + } + stage('E2E Tests') { + steps { + sh 'npm run test:e2e' + } + } + } + } + } +} +``` + +## Shared Libraries + +### Library Structure + +``` +vars/ +├── buildApp.groovy +├── deployApp.groovy +└── notifySlack.groovy +src/ +└── com/example/ + └── Pipeline.groovy +resources/ +└── templates/ + └── deployment.yaml +``` + +### Define Shared Step + +```groovy +// vars/buildApp.groovy +def call(Map config = [:]) { + def nodeVersion = config.nodeVersion ?: '20' + + docker.image("node:${nodeVersion}").inside { + sh 'npm ci' + sh 'npm run build' + } +} +``` + +### Use Shared Library + +```groovy +@Library('my-shared-library') _ + +pipeline { + agent any + + stages { + stage('Build') { + steps { + buildApp(nodeVersion: '20') + } + } + stage('Deploy') { + steps { + deployApp(environment: 'staging') + } + } + } + + post { + failure { + notifySlack(channel: '#builds', status: 'FAILED') + } + } +} +``` + +## Scripted Pipeline + +```groovy +node('linux') { + try { + stage('Checkout') { + checkout scm + } + + stage('Build') { + docker.image('node:20').inside { + sh 'npm ci' + sh 'npm run build' + } + } + + stage('Test') { + sh 'npm test' + } + + if (env.BRANCH_NAME == 'main') { + stage('Deploy') { + sh './deploy.sh' + } + } + } catch (e) { + currentBuild.result = 'FAILURE' + throw e + } finally { + cleanWs() + } +} +``` + +## Plugin Management + +### Essential Plugins + +```groovy +// Install via Jenkins CLI or init.groovy.d +def plugins = [ + 'workflow-aggregator', // Pipeline + 'git', // Git integration + 'docker-workflow', // Docker Pipeline + 'kubernetes', // Kubernetes agent + 'credentials-binding', // Credentials + 'blueocean', // Blue Ocean UI + 'job-dsl', // Job DSL + 'configuration-as-code' // JCasC +] +``` + +### Configuration as Code + +```yaml +# jenkins.yaml +jenkins: + systemMessage: "Jenkins configured via JCasC" + numExecutors: 2 + + securityRealm: + local: + users: + - id: admin + password: ${ADMIN_PASSWORD} + + authorizationStrategy: + globalMatrix: + permissions: + - "Overall/Administer:admin" + - "Overall/Read:authenticated" + +credentials: + system: + domainCredentials: + - credentials: + - usernamePassword: + id: "docker-hub" + username: "user" + password: ${DOCKER_PASSWORD} +``` + +## Multibranch Pipeline + +```groovy +// Automatically discovers branches with Jenkinsfile +// Configure in Jenkins UI: New Item > Multibranch Pipeline + +// Branch-specific behavior in Jenkinsfile +pipeline { + agent any + + stages { + stage('Deploy') { + when { + anyOf { + branch 'main' + branch 'release/*' + } + } + steps { + sh './deploy.sh' + } + } + } +} +``` + +## Common Issues + +### Issue: Pipeline Syntax Errors +**Problem**: Jenkinsfile fails to parse +**Solution**: Use Pipeline Syntax generator in Jenkins UI, validate with `jenkins-cli` + +### Issue: Agent Not Connecting +**Problem**: Build agents disconnect +**Solution**: Check agent logs, verify network connectivity, increase timeout settings + +### Issue: Out of Memory +**Problem**: Jenkins crashes or builds fail with OOM +**Solution**: Increase heap size in `JAVA_OPTS`, clean up old builds + +## Best Practices + +- Use declarative pipelines for most use cases +- Implement shared libraries for reusable code +- Store Jenkinsfile in source control +- Use credentials plugin for secrets management +- Implement proper cleanup in post blocks +- Configure build retention policies +- Use Blue Ocean for modern UI experience + +## Related Skills + +- [github-actions](../github-actions/) - GitHub native CI/CD +- [kubernetes-ops](../../orchestration/kubernetes-ops/) - K8s deployment target +- [docker-management](../../containers/docker-management/) - Container builds diff --git a/devops/ci-cd/jenkins/references/pipeline-syntax.md b/devops/ci-cd/jenkins/references/pipeline-syntax.md new file mode 100644 index 0000000..694f34e --- /dev/null +++ b/devops/ci-cd/jenkins/references/pipeline-syntax.md @@ -0,0 +1,128 @@ +# Jenkins Pipeline Syntax Reference + +## Declarative Pipeline + +```groovy +pipeline { + agent any + + options { + timeout(time: 1, unit: 'HOURS') + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '10')) + } + + environment { + DEPLOY_ENV = 'production' + CREDS = credentials('my-credentials') + } + + stages { + stage('Build') { + steps { + sh 'make build' + } + } + + stage('Test') { + parallel { + stage('Unit Tests') { + steps { + sh 'make test-unit' + } + } + stage('Integration Tests') { + steps { + sh 'make test-integration' + } + } + } + } + + stage('Deploy') { + when { + branch 'main' + } + steps { + sh 'make deploy' + } + } + } + + post { + always { + junit 'reports/**/*.xml' + cleanWs() + } + success { + slackSend color: 'good', message: 'Build succeeded' + } + failure { + slackSend color: 'danger', message: 'Build failed' + } + } +} +``` + +## Scripted Pipeline + +```groovy +node('linux') { + try { + stage('Checkout') { + checkout scm + } + + stage('Build') { + sh 'make build' + } + + if (env.BRANCH_NAME == 'main') { + stage('Deploy') { + sh 'make deploy' + } + } + } catch (e) { + currentBuild.result = 'FAILURE' + throw e + } finally { + cleanWs() + } +} +``` + +## Shared Libraries + +```groovy +// vars/buildPipeline.groovy +def call(Map config) { + pipeline { + agent any + stages { + stage('Build') { + steps { + sh config.buildCommand ?: 'make build' + } + } + } + } +} + +// Jenkinsfile +@Library('my-shared-library') _ +buildPipeline(buildCommand: 'npm run build') +``` + +## Credentials + +```groovy +withCredentials([ + usernamePassword( + credentialsId: 'docker-hub', + usernameVariable: 'DOCKER_USER', + passwordVariable: 'DOCKER_PASS' + ) +]) { + sh 'docker login -u $DOCKER_USER -p $DOCKER_PASS' +} +``` diff --git a/devops/containers/container-registries/SKILL.md b/devops/containers/container-registries/SKILL.md new file mode 100644 index 0000000..b07a3b1 --- /dev/null +++ b/devops/containers/container-registries/SKILL.md @@ -0,0 +1,411 @@ +--- +name: container-registries +description: Manage container registries including ECR, ACR, GCR, and Docker Hub. Push and pull images, configure authentication, set up repository policies, and implement image lifecycle management. Use when working with container image storage and distribution. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# Container Registries + +Store, manage, and distribute container images across cloud and self-hosted registries. + +## When to Use This Skill + +Use this skill when: +- Pushing and pulling container images +- Configuring registry authentication +- Setting up image retention policies +- Managing private container registries +- Implementing image scanning and security + +## Prerequisites + +- Docker or Podman installed +- Cloud CLI tools (AWS CLI, az, gcloud) for respective registries +- Appropriate IAM permissions + +## Docker Hub + +### Authentication + +```bash +# Login +docker login + +# Login with token +echo "$DOCKER_TOKEN" | docker login -u username --password-stdin +``` + +### Push/Pull Images + +```bash +# Tag image +docker tag myapp:latest username/myapp:latest + +# Push +docker push username/myapp:latest + +# Pull +docker pull username/myapp:latest +``` + +### Automated Builds + +Configure in Docker Hub UI: +1. Connect GitHub/Bitbucket repository +2. Set build rules (branch → tag mapping) +3. Configure build context and Dockerfile path + +## Amazon ECR + +### Setup + +```bash +# Create repository +aws ecr create-repository \ + --repository-name myapp \ + --image-scanning-configuration scanOnPush=true \ + --encryption-configuration encryptionType=AES256 + +# Get registry URI +REGISTRY=$(aws ecr describe-repositories \ + --repository-names myapp \ + --query 'repositories[0].repositoryUri' \ + --output text | cut -d'/' -f1) +``` + +### Authentication + +```bash +# Login (Docker) +aws ecr get-login-password --region us-east-1 | \ + docker login --username AWS --password-stdin $REGISTRY + +# Login with credential helper +# Add to ~/.docker/config.json: +{ + "credHelpers": { + "123456789.dkr.ecr.us-east-1.amazonaws.com": "ecr-login" + } +} +``` + +### Push/Pull + +```bash +# Tag and push +docker tag myapp:latest $REGISTRY/myapp:latest +docker push $REGISTRY/myapp:latest + +# Pull +docker pull $REGISTRY/myapp:latest +``` + +### Lifecycle Policy + +```bash +# Create lifecycle policy +aws ecr put-lifecycle-policy \ + --repository-name myapp \ + --lifecycle-policy-text '{ + "rules": [ + { + "rulePriority": 1, + "description": "Keep last 10 images", + "selection": { + "tagStatus": "any", + "countType": "imageCountMoreThan", + "countNumber": 10 + }, + "action": { + "type": "expire" + } + } + ] + }' +``` + +### Repository Policy + +```bash +# Allow cross-account access +aws ecr set-repository-policy \ + --repository-name myapp \ + --policy-text '{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "CrossAccountPull", + "Effect": "Allow", + "Principal": { + "AWS": "arn:aws:iam::OTHER_ACCOUNT:root" + }, + "Action": [ + "ecr:GetDownloadUrlForLayer", + "ecr:BatchGetImage" + ] + } + ] + }' +``` + +## Azure Container Registry (ACR) + +### Setup + +```bash +# Create registry +az acr create \ + --resource-group mygroup \ + --name myregistry \ + --sku Standard \ + --admin-enabled false + +# Get login server +az acr show --name myregistry --query loginServer -o tsv +``` + +### Authentication + +```bash +# Login with Azure CLI +az acr login --name myregistry + +# Login with service principal +docker login myregistry.azurecr.io \ + -u $SP_APP_ID \ + -p $SP_PASSWORD + +# Get access token +az acr login --name myregistry --expose-token +``` + +### Push/Pull + +```bash +# Tag and push +docker tag myapp:latest myregistry.azurecr.io/myapp:latest +docker push myregistry.azurecr.io/myapp:latest + +# ACR Build (build in cloud) +az acr build \ + --registry myregistry \ + --image myapp:latest \ + --file Dockerfile . +``` + +### Retention Policy + +```bash +# Enable retention policy +az acr config retention update \ + --registry myregistry \ + --status enabled \ + --days 30 \ + --type UntaggedManifests +``` + +### Geo-Replication + +```bash +# Enable replication +az acr replication create \ + --registry myregistry \ + --location westeurope + +# List replications +az acr replication list --registry myregistry +``` + +## Google Container Registry (GCR) / Artifact Registry + +### Setup (Artifact Registry) + +```bash +# Create repository +gcloud artifacts repositories create myrepo \ + --repository-format=docker \ + --location=us-central1 \ + --description="Docker repository" +``` + +### Authentication + +```bash +# Configure Docker auth +gcloud auth configure-docker us-central1-docker.pkg.dev + +# Or use credential helper +gcloud auth print-access-token | \ + docker login -u oauth2accesstoken --password-stdin \ + https://us-central1-docker.pkg.dev +``` + +### Push/Pull + +```bash +# Tag for Artifact Registry +docker tag myapp:latest \ + us-central1-docker.pkg.dev/PROJECT_ID/myrepo/myapp:latest + +# Push +docker push us-central1-docker.pkg.dev/PROJECT_ID/myrepo/myapp:latest + +# Pull +docker pull us-central1-docker.pkg.dev/PROJECT_ID/myrepo/myapp:latest +``` + +### Cleanup Policy + +```bash +# Create cleanup policy +gcloud artifacts repositories set-cleanup-policies myrepo \ + --location=us-central1 \ + --policy=policy.json + +# policy.json +{ + "name": "delete-old", + "action": {"type": "Delete"}, + "condition": { + "olderThan": "30d", + "tagState": "untagged" + } +} +``` + +## GitHub Container Registry (GHCR) + +### Authentication + +```bash +# Login with PAT +echo "$GITHUB_TOKEN" | docker login ghcr.io -u USERNAME --password-stdin +``` + +### Push/Pull + +```bash +# Tag +docker tag myapp:latest ghcr.io/OWNER/myapp:latest + +# Push +docker push ghcr.io/OWNER/myapp:latest + +# Pull +docker pull ghcr.io/OWNER/myapp:latest +``` + +### Visibility Settings + +Configure in GitHub: +1. Go to package settings +2. Change visibility (public/private) +3. Manage access for teams/users + +## Self-Hosted Registry + +### Deploy with Docker + +```bash +# Run registry +docker run -d -p 5000:5000 \ + --name registry \ + -v registry-data:/var/lib/registry \ + registry:2 + +# Configure TLS +docker run -d -p 443:5000 \ + --name registry \ + -v /certs:/certs \ + -v registry-data:/var/lib/registry \ + -e REGISTRY_HTTP_TLS_CERTIFICATE=/certs/domain.crt \ + -e REGISTRY_HTTP_TLS_KEY=/certs/domain.key \ + registry:2 +``` + +### Harbor Registry + +```bash +# Download Harbor +wget https://github.com/goharbor/harbor/releases/download/v2.9.0/harbor-online-installer-v2.9.0.tgz +tar xzvf harbor-online-installer-v2.9.0.tgz + +# Configure harbor.yml +# Set hostname, https certificate, admin password + +# Install +./install.sh --with-trivy --with-chartmuseum +``` + +## Image Security + +### Vulnerability Scanning + +```bash +# ECR - Enable scan on push +aws ecr put-image-scanning-configuration \ + --repository-name myapp \ + --image-scanning-configuration scanOnPush=true + +# Get scan results +aws ecr describe-image-scan-findings \ + --repository-name myapp \ + --image-id imageTag=latest + +# ACR - Scan with Defender +az acr task create \ + --registry myregistry \ + --name scan-images \ + --cmd "mcr.microsoft.com/azure-cli az acr run-scan" +``` + +### Image Signing + +```bash +# Enable content trust +export DOCKER_CONTENT_TRUST=1 + +# Sign image on push +docker push myregistry/myapp:latest + +# Verify signature +docker trust inspect myregistry/myapp:latest +``` + +## Common Issues + +### Issue: Authentication Expired +**Problem**: Push/pull fails with auth error +**Solution**: Re-run login command, check credential helper + +### Issue: Image Not Found +**Problem**: Pull fails with manifest unknown +**Solution**: Verify tag exists, check registry URL + +### Issue: Push Permission Denied +**Problem**: Cannot push to repository +**Solution**: Check IAM permissions, verify repository exists + +### Issue: Rate Limiting (Docker Hub) +**Problem**: Too many requests error +**Solution**: Authenticate for higher limits, use pull-through cache + +## Best Practices + +- Enable vulnerability scanning on all repositories +- Implement lifecycle policies to manage storage costs +- Use immutable tags for production images +- Configure cross-region replication for availability +- Use service accounts/principals for CI/CD authentication +- Enable audit logging for compliance +- Implement image signing for supply chain security +- Use pull-through cache to avoid rate limits + +## Related Skills + +- [docker-management](../docker-management/) - Building images +- [container-scanning](../../../security/scanning/container-scanning/) - Security scanning +- [aws-iam](../../../infrastructure/cloud-aws/aws-iam/) - AWS permissions diff --git a/devops/containers/docker-compose/SKILL.md b/devops/containers/docker-compose/SKILL.md new file mode 100644 index 0000000..aa9fe76 --- /dev/null +++ b/devops/containers/docker-compose/SKILL.md @@ -0,0 +1,458 @@ +--- +name: docker-compose +description: Define and run multi-container Docker applications using Docker Compose. Create compose files, manage service dependencies, configure networks and volumes, and orchestrate local development environments. Use when setting up multi-service applications or development environments. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# Docker Compose + +Orchestrate multi-container applications with declarative YAML configuration. + +## When to Use This Skill + +Use this skill when: +- Running multi-container applications locally +- Setting up development environments +- Defining service dependencies and networking +- Managing application stacks with multiple services +- Creating reproducible development setups + +## Prerequisites + +- Docker Engine with Compose plugin (v2) +- Basic Docker knowledge +- YAML syntax understanding + +## Basic Configuration + +### Simple Application Stack + +```yaml +# docker-compose.yml +version: '3.8' + +services: + web: + build: . + ports: + - "3000:3000" + environment: + - NODE_ENV=development + - DATABASE_URL=postgres://postgres:secret@db:5432/myapp + depends_on: + - db + - redis + + db: + image: postgres:15-alpine + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: secret + POSTGRES_DB: myapp + volumes: + - postgres-data:/var/lib/postgresql/data + ports: + - "5432:5432" + + redis: + image: redis:7-alpine + ports: + - "6379:6379" + +volumes: + postgres-data: +``` + +## Service Configuration + +### Build Options + +```yaml +services: + app: + build: + context: ./app + dockerfile: Dockerfile.dev + args: + NODE_VERSION: "20" + target: development + cache_from: + - myapp:cache + image: myapp:dev +``` + +### Environment Variables + +```yaml +services: + app: + environment: + - NODE_ENV=production + - API_KEY=${API_KEY} # From shell or .env file + env_file: + - .env + - .env.local +``` + +### Port Mapping + +```yaml +services: + web: + ports: + - "3000:3000" # HOST:CONTAINER + - "127.0.0.1:9229:9229" # Bind to localhost only + - "8080-8090:8080-8090" # Port range + expose: + - "3000" # Internal only (no host binding) +``` + +### Volume Mounts + +```yaml +services: + app: + volumes: + # Named volume + - app-data:/app/data + # Bind mount + - ./src:/app/src + # Read-only bind mount + - ./config:/app/config:ro + # Anonymous volume (for node_modules) + - /app/node_modules + +volumes: + app-data: + driver: local +``` + +### Dependencies + +```yaml +services: + web: + depends_on: + db: + condition: service_healthy + redis: + condition: service_started + + db: + image: postgres:15 + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres"] + interval: 10s + timeout: 5s + retries: 5 +``` + +## Networking + +### Custom Networks + +```yaml +services: + frontend: + networks: + - frontend-net + + backend: + networks: + - frontend-net + - backend-net + + db: + networks: + - backend-net + +networks: + frontend-net: + driver: bridge + backend-net: + driver: bridge + internal: true # No external access +``` + +### Network Aliases + +```yaml +services: + db: + networks: + backend: + aliases: + - database + - postgres + +networks: + backend: +``` + +## Resource Limits + +```yaml +services: + app: + deploy: + resources: + limits: + cpus: '2' + memory: 1G + reservations: + cpus: '0.5' + memory: 256M +``` + +## Multiple Compose Files + +### Override Files + +```yaml +# docker-compose.yml (base) +services: + web: + image: myapp:latest + ports: + - "3000:3000" + +# docker-compose.override.yml (development - auto-loaded) +services: + web: + build: . + volumes: + - ./src:/app/src + environment: + - DEBUG=true + +# docker-compose.prod.yml (production) +services: + web: + deploy: + replicas: 3 + environment: + - DEBUG=false +``` + +### Using Multiple Files + +```bash +# Development (uses override automatically) +docker compose up + +# Production +docker compose -f docker-compose.yml -f docker-compose.prod.yml up + +# Merge and view final config +docker compose -f docker-compose.yml -f docker-compose.prod.yml config +``` + +## Profiles + +```yaml +services: + web: + image: myapp + + db: + image: postgres:15 + + debug: + image: busybox + profiles: + - debug + + monitoring: + image: prometheus + profiles: + - monitoring +``` + +```bash +# Run without profiles (web, db only) +docker compose up + +# Run with debug profile +docker compose --profile debug up + +# Run with multiple profiles +docker compose --profile debug --profile monitoring up +``` + +## Commands + +### Lifecycle + +```bash +# Start services +docker compose up -d + +# Start specific service +docker compose up -d web + +# Stop services +docker compose stop + +# Stop and remove containers +docker compose down + +# Stop and remove everything including volumes +docker compose down -v --rmi all + +# Restart services +docker compose restart web +``` + +### Building + +```bash +# Build images +docker compose build + +# Build without cache +docker compose build --no-cache + +# Build and start +docker compose up --build + +# Pull latest images +docker compose pull +``` + +### Monitoring + +```bash +# View logs +docker compose logs -f + +# View specific service logs +docker compose logs -f web + +# View running services +docker compose ps + +# View resource usage +docker compose top +``` + +### Execution + +```bash +# Run command in new container +docker compose run --rm web npm test + +# Execute in running container +docker compose exec web /bin/sh + +# Scale service +docker compose up -d --scale worker=3 +``` + +## Development Workflow + +### Watch Mode (Compose v2.22+) + +```yaml +services: + web: + build: . + develop: + watch: + - action: sync + path: ./src + target: /app/src + - action: rebuild + path: ./package.json +``` + +```bash +docker compose watch +``` + +### Hot Reload Setup + +```yaml +services: + web: + build: + context: . + target: development + volumes: + - ./src:/app/src + - /app/node_modules + environment: + - CHOKIDAR_USEPOLLING=true + command: npm run dev +``` + +## Common Patterns + +### Database Initialization + +```yaml +services: + db: + image: postgres:15 + volumes: + - postgres-data:/var/lib/postgresql/data + - ./init-scripts:/docker-entrypoint-initdb.d:ro + environment: + POSTGRES_DB: myapp +``` + +### Reverse Proxy + +```yaml +services: + proxy: + image: traefik:v3.0 + ports: + - "80:80" + - "443:443" + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + - ./traefik.yml:/etc/traefik/traefik.yml:ro + + web: + labels: + - "traefik.enable=true" + - "traefik.http.routers.web.rule=Host(`app.localhost`)" +``` + +## Common Issues + +### Issue: Container Cannot Resolve Service Name +**Problem**: Service can't connect to another service by name +**Solution**: Ensure services are on the same network, check depends_on + +### Issue: Volume Permissions +**Problem**: Container can't write to mounted volume +**Solution**: Match container user UID with host, or use named volumes + +### Issue: Port Already in Use +**Problem**: Error binding to port +**Solution**: Change host port or stop conflicting service + +### Issue: Changes Not Reflected +**Problem**: Code changes don't appear in container +**Solution**: Check volume mounts, rebuild if Dockerfile changed + +## Best Practices + +- Use named volumes for persistent data +- Define healthchecks for database dependencies +- Use profiles to separate optional services +- Keep secrets in .env files (not committed) +- Use override files for environment-specific config +- Pin image versions for reproducibility +- Use networks to isolate service groups +- Leverage watch mode for development + +## Related Skills + +- [docker-management](../docker-management/) - Docker fundamentals +- [kubernetes-ops](../../orchestration/kubernetes-ops/) - Production orchestration +- [reverse-proxy](../../../infrastructure/networking/reverse-proxy/) - Production routing diff --git a/devops/containers/docker-compose/references/compose-patterns.md b/devops/containers/docker-compose/references/compose-patterns.md new file mode 100644 index 0000000..e90d59c --- /dev/null +++ b/devops/containers/docker-compose/references/compose-patterns.md @@ -0,0 +1,124 @@ +# Docker Compose Patterns + +## Basic Structure + +```yaml +version: '3.8' + +services: + app: + build: . + ports: + - "8080:80" + environment: + - NODE_ENV=production + depends_on: + - db + networks: + - frontend + - backend + + db: + image: postgres:15 + volumes: + - db_data:/var/lib/postgresql/data + environment: + POSTGRES_DB: myapp + POSTGRES_PASSWORD_FILE: /run/secrets/db_password + secrets: + - db_password + networks: + - backend + +volumes: + db_data: + +networks: + frontend: + backend: + +secrets: + db_password: + file: ./secrets/db_password.txt +``` + +## Health Checks + +```yaml +services: + app: + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost/health"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 40s +``` + +## Resource Limits + +```yaml +services: + app: + deploy: + resources: + limits: + cpus: '0.5' + memory: 512M + reservations: + cpus: '0.25' + memory: 256M +``` + +## Multiple Compose Files + +```bash +# Base + overrides +docker compose -f docker-compose.yml -f docker-compose.prod.yml up + +# Or use COMPOSE_FILE +export COMPOSE_FILE=docker-compose.yml:docker-compose.prod.yml +docker compose up +``` + +## Environment Variables + +```yaml +# .env file +services: + app: + image: myapp:${VERSION:-latest} + environment: + - DATABASE_URL=${DATABASE_URL} + env_file: + - .env + - .env.local +``` + +## Service Profiles + +```yaml +services: + app: + profiles: [] # Always starts + + debug: + profiles: [debug] + # Only starts with --profile debug +``` + +## Extension Fields + +```yaml +x-common: &common + restart: unless-stopped + logging: + driver: json-file + options: + max-size: "10m" + +services: + app: + <<: *common + image: myapp +``` diff --git a/devops/containers/docker-management/SKILL.md b/devops/containers/docker-management/SKILL.md new file mode 100644 index 0000000..b47227a --- /dev/null +++ b/devops/containers/docker-management/SKILL.md @@ -0,0 +1,402 @@ +--- +name: docker-management +description: Build, optimize, and troubleshoot Docker containers and images. Create efficient Dockerfiles, manage container lifecycle, configure networking and volumes, and debug container issues. Use when working with Docker, containerization, or container troubleshooting. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# Docker Management + +Build, run, and manage Docker containers for application deployment and development. + +## When to Use This Skill + +Use this skill when: +- Creating and optimizing Dockerfiles +- Building and tagging Docker images +- Running and managing containers +- Debugging container issues +- Configuring Docker networking and volumes +- Implementing container security best practices + +## Prerequisites + +- Docker Engine installed (20.10+) +- Basic command line knowledge +- Understanding of application deployment + +## Dockerfile Best Practices + +### Multi-Stage Build + +```dockerfile +# Build stage +FROM node:20-alpine AS builder +WORKDIR /app +COPY package*.json ./ +RUN npm ci --only=production +COPY . . +RUN npm run build + +# Production stage +FROM node:20-alpine AS production +WORKDIR /app +RUN addgroup -g 1001 -S nodejs && \ + adduser -S nodejs -u 1001 +COPY --from=builder --chown=nodejs:nodejs /app/dist ./dist +COPY --from=builder --chown=nodejs:nodejs /app/node_modules ./node_modules +USER nodejs +EXPOSE 3000 +CMD ["node", "dist/index.js"] +``` + +### Layer Optimization + +```dockerfile +FROM python:3.12-slim + +# Install dependencies first (cached unless requirements change) +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +# Copy application code (changes frequently) +COPY . . + +CMD ["python", "app.py"] +``` + +### Security Hardening + +```dockerfile +FROM node:20-alpine + +# Create non-root user +RUN addgroup -g 1001 appgroup && \ + adduser -u 1001 -G appgroup -D appuser + +WORKDIR /app + +# Copy with proper ownership +COPY --chown=appuser:appgroup . . + +# Drop privileges +USER appuser + +# Use exec form for proper signal handling +CMD ["node", "server.js"] +``` + +## Building Images + +### Basic Build + +```bash +# Build with tag +docker build -t myapp:1.0 . + +# Build with build args +docker build --build-arg NODE_ENV=production -t myapp:prod . + +# Build for specific platform +docker build --platform linux/amd64 -t myapp:amd64 . + +# Build with no cache +docker build --no-cache -t myapp:fresh . +``` + +### Multi-Platform Builds + +```bash +# Create builder +docker buildx create --name multiplatform --use + +# Build for multiple architectures +docker buildx build \ + --platform linux/amd64,linux/arm64 \ + -t myregistry/myapp:latest \ + --push . +``` + +## Running Containers + +### Basic Operations + +```bash +# Run container +docker run -d --name myapp -p 8080:3000 myapp:latest + +# Run with environment variables +docker run -d \ + -e DATABASE_URL=postgres://localhost/db \ + -e NODE_ENV=production \ + myapp:latest + +# Run with resource limits +docker run -d \ + --memory="512m" \ + --cpus="1.0" \ + myapp:latest + +# Run with restart policy +docker run -d --restart=unless-stopped myapp:latest +``` + +### Volume Management + +```bash +# Named volume +docker volume create mydata +docker run -v mydata:/app/data myapp:latest + +# Bind mount +docker run -v $(pwd)/config:/app/config:ro myapp:latest + +# tmpfs mount (memory) +docker run --tmpfs /tmp:rw,noexec,nosuid myapp:latest +``` + +### Networking + +```bash +# Create network +docker network create mynetwork + +# Run on network +docker run -d --network mynetwork --name api myapp:latest + +# Connect existing container +docker network connect mynetwork existing-container + +# Expose specific ports +docker run -d -p 127.0.0.1:8080:3000 myapp:latest +``` + +## Container Lifecycle + +### Management Commands + +```bash +# List containers +docker ps -a + +# Stop container +docker stop myapp + +# Remove container +docker rm myapp + +# Force remove running container +docker rm -f myapp + +# Prune stopped containers +docker container prune -f +``` + +### Logs and Monitoring + +```bash +# View logs +docker logs myapp + +# Follow logs +docker logs -f --tail 100 myapp + +# View resource usage +docker stats myapp + +# Inspect container +docker inspect myapp +``` + +## Debugging Containers + +### Interactive Access + +```bash +# Execute command in running container +docker exec -it myapp /bin/sh + +# Run container with shell +docker run -it --rm myapp:latest /bin/sh + +# Debug failed container +docker run -it --entrypoint /bin/sh myapp:latest +``` + +### Troubleshooting + +```bash +# Check container logs for errors +docker logs myapp 2>&1 | grep -i error + +# Inspect container state +docker inspect --format='{{.State.Status}}' myapp + +# Check container processes +docker top myapp + +# View container filesystem changes +docker diff myapp + +# Export container filesystem +docker export myapp > myapp-fs.tar +``` + +### Health Checks + +```dockerfile +HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \ + CMD curl -f http://localhost:3000/health || exit 1 +``` + +```bash +# Check health status +docker inspect --format='{{.State.Health.Status}}' myapp +``` + +## Image Management + +### Tagging and Pushing + +```bash +# Tag image +docker tag myapp:latest myregistry.com/myapp:v1.0 + +# Push to registry +docker push myregistry.com/myapp:v1.0 + +# Pull image +docker pull myregistry.com/myapp:v1.0 +``` + +### Cleanup + +```bash +# Remove unused images +docker image prune -a + +# Remove all unused resources +docker system prune -a --volumes + +# Remove specific image +docker rmi myapp:old + +# List image sizes +docker images --format "table {{.Repository}}\t{{.Tag}}\t{{.Size}}" +``` + +### Image Analysis + +```bash +# View image history +docker history myapp:latest + +# Inspect image layers +docker inspect myapp:latest + +# Check image vulnerabilities (with Docker Scout) +docker scout cves myapp:latest +``` + +## Docker Compose Integration + +```yaml +# docker-compose.yml +version: '3.8' + +services: + app: + build: + context: . + dockerfile: Dockerfile + ports: + - "3000:3000" + environment: + - NODE_ENV=production + volumes: + - app-data:/app/data + depends_on: + - db + restart: unless-stopped + + db: + image: postgres:15-alpine + environment: + POSTGRES_PASSWORD: secret + volumes: + - db-data:/var/lib/postgresql/data + +volumes: + app-data: + db-data: +``` + +## Security Best Practices + +### Image Security + +```dockerfile +# Use specific version tags +FROM node:20.10-alpine3.18 + +# Don't run as root +USER nobody + +# Remove unnecessary packages +RUN apk del --purge build-dependencies + +# Use COPY instead of ADD +COPY . . +``` + +### Runtime Security + +```bash +# Run with security options +docker run -d \ + --security-opt=no-new-privileges \ + --cap-drop=ALL \ + --cap-add=NET_BIND_SERVICE \ + --read-only \ + myapp:latest + +# Use user namespace remapping +# Add to /etc/docker/daemon.json: {"userns-remap": "default"} +``` + +## Common Issues + +### Issue: Container Exits Immediately +**Problem**: Container starts and stops instantly +**Solution**: Check if CMD/ENTRYPOINT runs foreground process, use `docker logs` to see errors + +### Issue: Cannot Connect to Container +**Problem**: Port not accessible +**Solution**: Verify port mapping (-p), check container is running, verify firewall rules + +### Issue: Out of Disk Space +**Problem**: Docker using too much disk +**Solution**: Run `docker system prune -a --volumes`, check for large unused images + +### Issue: Build Cache Not Working +**Problem**: Every build downloads dependencies +**Solution**: Order Dockerfile instructions from least to most frequently changing + +## Best Practices + +- Use multi-stage builds to minimize image size +- Never store secrets in images - use runtime injection +- Pin base image versions for reproducibility +- Implement health checks for production containers +- Use .dockerignore to exclude unnecessary files +- Run containers as non-root users +- Scan images for vulnerabilities regularly +- Use Docker BuildKit for faster builds + +## Related Skills + +- [docker-compose](../docker-compose/) - Multi-container applications +- [container-scanning](../../../security/scanning/container-scanning/) - Security scanning +- [container-hardening](../../../security/hardening/container-hardening/) - Security hardening diff --git a/devops/containers/docker-management/references/docker-commands.md b/devops/containers/docker-management/references/docker-commands.md new file mode 100644 index 0000000..ab1a361 --- /dev/null +++ b/devops/containers/docker-management/references/docker-commands.md @@ -0,0 +1,111 @@ +# Docker Command Reference + +## Container Lifecycle + +```bash +# Run container +docker run -d --name myapp -p 8080:80 nginx +docker run -it --rm ubuntu bash + +# Start/Stop +docker start myapp +docker stop myapp +docker restart myapp + +# Remove +docker rm myapp +docker rm -f myapp # Force + +# Logs +docker logs myapp +docker logs -f myapp # Follow +docker logs --tail 100 myapp +``` + +## Images + +```bash +# List/Pull/Build +docker images +docker pull nginx:latest +docker build -t myapp:1.0 . +docker build -t myapp:1.0 -f Dockerfile.prod . + +# Tag/Push +docker tag myapp:1.0 registry.example.com/myapp:1.0 +docker push registry.example.com/myapp:1.0 + +# Remove +docker rmi myapp:1.0 +docker image prune -a # Remove unused +``` + +## Inspection + +```bash +# Container info +docker ps +docker ps -a +docker inspect myapp +docker stats +docker top myapp + +# Exec into container +docker exec -it myapp bash +docker exec myapp ls -la /app +``` + +## Networks + +```bash +# List/Create +docker network ls +docker network create mynet + +# Connect container +docker network connect mynet myapp +docker run --network mynet nginx +``` + +## Volumes + +```bash +# List/Create +docker volume ls +docker volume create mydata + +# Mount +docker run -v mydata:/data nginx +docker run -v $(pwd):/app nginx +docker run --mount type=bind,source=$(pwd),target=/app nginx +``` + +## Cleanup + +```bash +# Remove stopped containers +docker container prune + +# Remove unused images +docker image prune -a + +# Remove everything unused +docker system prune -a --volumes + +# Disk usage +docker system df +``` + +## Multi-stage Build + +```dockerfile +FROM node:20 AS builder +WORKDIR /app +COPY package*.json ./ +RUN npm ci +COPY . . +RUN npm run build + +FROM nginx:alpine +COPY --from=builder /app/dist /usr/share/nginx/html +``` diff --git a/devops/containers/podman/SKILL.md b/devops/containers/podman/SKILL.md new file mode 100644 index 0000000..51c7f50 --- /dev/null +++ b/devops/containers/podman/SKILL.md @@ -0,0 +1,381 @@ +--- +name: podman +description: Manage containers using Podman, the daemonless container engine. Run rootless containers, create pods, manage images, and use Docker-compatible commands. Use when working with Podman or requiring rootless container operations. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# Podman + +Run and manage containers without a daemon using Podman's rootless container engine. + +## When to Use This Skill + +Use this skill when: +- Running containers without root privileges +- Managing containers on systems without Docker +- Creating pod-based container groups +- Using systemd for container management +- Working in security-conscious environments + +## Prerequisites + +- Podman installed (4.x+) +- For rootless: user namespaces enabled +- Basic container concepts understanding + +## Key Differences from Docker + +| Feature | Docker | Podman | +|---------|--------|--------| +| Architecture | Client-daemon | Daemonless | +| Root required | Default | Optional (rootless) | +| Pod support | No | Yes (Kubernetes-style) | +| Systemd integration | Limited | Native | +| Socket | docker.sock | podman.sock (optional) | + +## Basic Commands + +### Container Operations + +```bash +# Run container (identical to Docker) +podman run -d --name webserver -p 8080:80 nginx + +# List containers +podman ps -a + +# Stop and remove +podman stop webserver +podman rm webserver + +# Execute command +podman exec -it webserver /bin/sh + +# View logs +podman logs -f webserver +``` + +### Image Management + +```bash +# Pull image +podman pull docker.io/library/nginx:latest + +# List images +podman images + +# Build image +podman build -t myapp:latest . + +# Push to registry +podman push myapp:latest registry.example.com/myapp:latest + +# Remove image +podman rmi nginx:latest +``` + +## Rootless Containers + +### Setup + +```bash +# Check user namespace support +cat /proc/sys/user/max_user_namespaces + +# Enable if needed (as root) +echo "user.max_user_namespaces=28633" | sudo tee /etc/sysctl.d/userns.conf +sudo sysctl -p /etc/sysctl.d/userns.conf + +# Configure subuid/subgid for user +sudo usermod --add-subuids 100000-165535 --add-subgids 100000-165535 $USER + +# Verify +podman unshare cat /proc/self/uid_map +``` + +### Running Rootless + +```bash +# Run as regular user (no sudo) +podman run -d --name myapp -p 8080:80 nginx + +# Check user namespace mapping +podman unshare id + +# Verify non-root +podman top myapp user +``` + +### Port Considerations + +```bash +# Rootless cannot bind to ports < 1024 by default +# Use ports >= 1024 +podman run -d -p 8080:80 nginx + +# Or enable unprivileged ports (as root) +echo "net.ipv4.ip_unprivileged_port_start=80" | sudo tee /etc/sysctl.d/ports.conf +sudo sysctl -p /etc/sysctl.d/ports.conf +``` + +## Pods + +### Creating Pods + +```bash +# Create pod +podman pod create --name mypod -p 8080:80 -p 5432:5432 + +# Add containers to pod +podman run -d --pod mypod --name web nginx +podman run -d --pod mypod --name db postgres:15 + +# List pods +podman pod ps + +# Containers share network namespace +podman exec web curl localhost:5432 +``` + +### Pod Management + +```bash +# Start/stop pod (affects all containers) +podman pod start mypod +podman pod stop mypod + +# Remove pod and containers +podman pod rm -f mypod + +# View pod details +podman pod inspect mypod + +# Generate Kubernetes YAML from pod +podman generate kube mypod > mypod.yaml +``` + +## Systemd Integration + +### Generate Systemd Unit + +```bash +# Generate unit file for container +podman generate systemd --new --name myapp > ~/.config/systemd/user/container-myapp.service + +# For pod +podman generate systemd --new --name mypod --files + +# Reload systemd +systemctl --user daemon-reload + +# Enable and start +systemctl --user enable --now container-myapp.service +``` + +### Quadlet (Podman 4.4+) + +```ini +# ~/.config/containers/systemd/webapp.container +[Container] +Image=docker.io/library/nginx:latest +PublishPort=8080:80 +Volume=webapp-data:/usr/share/nginx/html + +[Service] +Restart=always + +[Install] +WantedBy=default.target +``` + +```bash +# Reload to generate service +systemctl --user daemon-reload + +# Start the service +systemctl --user start webapp +``` + +## Compose Compatibility + +### Using Podman Compose + +```bash +# Install podman-compose +pip install podman-compose + +# Run compose file +podman-compose up -d + +# Or use Docker Compose with Podman socket +systemctl --user enable --now podman.socket +export DOCKER_HOST=unix:///run/user/$UID/podman/podman.sock +docker-compose up -d +``` + +### Native Podman Kube + +```bash +# Play Kubernetes YAML +podman kube play deployment.yaml + +# Stop and remove +podman kube down deployment.yaml +``` + +## Networking + +### Network Management + +```bash +# Create network +podman network create mynetwork + +# Run on network +podman run -d --network mynetwork --name app myapp + +# Connect container to network +podman network connect mynetwork existing-container + +# List networks +podman network ls + +# Inspect network +podman network inspect mynetwork +``` + +### DNS Resolution + +```bash +# Containers on same network can resolve by name +podman run -d --network mynetwork --name db postgres:15 +podman run -d --network mynetwork --name app \ + -e DATABASE_HOST=db myapp +``` + +## Storage + +### Volume Management + +```bash +# Create volume +podman volume create mydata + +# Use volume +podman run -d -v mydata:/data myapp + +# List volumes +podman volume ls + +# Inspect volume +podman volume inspect mydata + +# Rootless volumes location +ls ~/.local/share/containers/storage/volumes/ +``` + +### Bind Mounts + +```bash +# Bind mount with SELinux label +podman run -v ./data:/app/data:Z myapp + +# Z = private label (single container) +# z = shared label (multiple containers) +``` + +## Registry Configuration + +### Configure Registries + +```bash +# Edit registries.conf +# ~/.config/containers/registries.conf +``` + +```toml +unqualified-search-registries = ["docker.io", "quay.io"] + +[[registry]] +prefix = "docker.io" +location = "docker.io" + +[[registry.mirror]] +location = "mirror.gcr.io" +``` + +### Authentication + +```bash +# Login to registry +podman login docker.io + +# Login to private registry +podman login registry.example.com + +# Credentials stored in +# ~/.config/containers/auth.json +``` + +## Building Images + +### Buildah Integration + +```bash +# Podman uses Buildah for builds +podman build -t myapp:latest . + +# Build with specific format +podman build --format docker -t myapp . + +# Multi-stage build +podman build --target production -t myapp:prod . +``` + +### Buildah Commands + +```bash +# Create container from scratch +buildah from scratch +buildah copy working-container ./app /app +buildah config --entrypoint '["/app/main"]' working-container +buildah commit working-container myapp:minimal +``` + +## Common Issues + +### Issue: Permission Denied +**Problem**: Cannot access files in mounted volumes +**Solution**: Use `:Z` or `:z` suffix for SELinux, or check ownership + +### Issue: Cannot Connect to Container +**Problem**: Port not accessible in rootless mode +**Solution**: Use ports >= 1024 or configure unprivileged port start + +### Issue: Slow Image Pulls +**Problem**: Images download slowly +**Solution**: Configure registry mirrors in registries.conf + +### Issue: Systemd Service Fails +**Problem**: Container doesn't start via systemd +**Solution**: Enable lingering: `loginctl enable-linger $USER` + +## Best Practices + +- Use rootless mode for enhanced security +- Leverage pods for related containers +- Generate systemd units for production +- Use Quadlet for declarative container services +- Configure SELinux labels for bind mounts +- Enable user lingering for persistent services +- Use podman auto-update for automatic updates +- Alias `docker` to `podman` for compatibility + +## Related Skills + +- [docker-management](../docker-management/) - Docker fundamentals +- [kubernetes-ops](../../orchestration/kubernetes-ops/) - K8s orchestration +- [container-hardening](../../../security/hardening/container-hardening/) - Security diff --git a/devops/observability/alerting-oncall/SKILL.md b/devops/observability/alerting-oncall/SKILL.md new file mode 100644 index 0000000..0a600c2 --- /dev/null +++ b/devops/observability/alerting-oncall/SKILL.md @@ -0,0 +1,498 @@ +--- +name: alerting-oncall +description: Set up alerting rules, configure on-call rotations, and manage incident response workflows. Integrate with PagerDuty, Opsgenie, or Grafana OnCall for alert routing and escalation. Use when implementing alerting strategies and on-call management for production systems. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# Alerting & On-Call + +Configure effective alerting and on-call management for production systems. + +## When to Use This Skill + +Use this skill when: +- Setting up alerting rules and thresholds +- Configuring on-call rotations and schedules +- Implementing alert routing and escalation +- Reducing alert fatigue +- Managing incident response workflows + +## Prerequisites + +- Monitoring system (Prometheus, Datadog, etc.) +- On-call platform (PagerDuty, Opsgenie, Grafana OnCall) +- Communication channels (Slack, email) + +## Alerting Best Practices + +### Alert Categories + +```yaml +# Severity levels +critical: + - Service completely down + - Data loss imminent + - Security breach + response: Immediate page, wake people up + +high: + - Service degraded significantly + - Error rate above SLO + - Capacity near limit + response: Page during business hours, notify after hours + +medium: + - Performance degradation + - Non-critical component failure + - Warning thresholds exceeded + response: Notify via Slack, review next business day + +low: + - Informational alerts + - Capacity planning triggers + - Routine maintenance needed + response: Email notification, weekly review +``` + +### Alert Design Principles + +```yaml +# Good alert characteristics +alerts: + actionable: + - Every alert should require human action + - Include runbook links + - Clear remediation steps + + relevant: + - Alert on symptoms, not causes + - Focus on user impact + - Avoid alerting on expected behavior + + timely: + - Appropriate thresholds + - Suitable evaluation windows + - Account for normal variance + + unique: + - No duplicate alerts + - Proper alert grouping + - Clear ownership +``` + +## Prometheus Alerting + +### Alert Rules + +```yaml +# prometheus/rules/alerts.yml +groups: + - name: service_alerts + rules: + # High-level service health + - alert: ServiceDown + expr: up{job="myapp"} == 0 + for: 1m + labels: + severity: critical + annotations: + summary: "Service {{ $labels.instance }} is down" + description: "{{ $labels.job }} on {{ $labels.instance }} has been down for more than 1 minute." + runbook_url: "https://wiki.example.com/runbooks/service-down" + + # Error rate alert + - alert: HighErrorRate + expr: | + sum(rate(http_requests_total{status=~"5.."}[5m])) by (service) + / sum(rate(http_requests_total[5m])) by (service) > 0.05 + for: 5m + labels: + severity: critical + annotations: + summary: "High error rate for {{ $labels.service }}" + description: "Error rate is {{ $value | humanizePercentage }} for the last 5 minutes" + + # Latency alert (SLO-based) + - alert: HighLatency + expr: | + histogram_quantile(0.95, + sum(rate(http_request_duration_seconds_bucket[5m])) by (le, service) + ) > 0.5 + for: 5m + labels: + severity: high + annotations: + summary: "P95 latency above 500ms for {{ $labels.service }}" +``` + +### Alertmanager Configuration + +```yaml +# alertmanager.yml +global: + resolve_timeout: 5m + slack_api_url: 'https://hooks.slack.com/services/xxx' + pagerduty_url: 'https://events.pagerduty.com/v2/enqueue' + +templates: + - '/etc/alertmanager/templates/*.tmpl' + +route: + receiver: 'default-receiver' + group_by: ['alertname', 'service'] + group_wait: 30s + group_interval: 5m + repeat_interval: 4h + + routes: + # Critical alerts go to PagerDuty + - match: + severity: critical + receiver: 'pagerduty-critical' + group_wait: 0s + repeat_interval: 1h + + # High severity during business hours + - match: + severity: high + receiver: 'slack-high' + active_time_intervals: + - business-hours + + # Route by team + - match_re: + team: platform.* + receiver: 'platform-team' + +receivers: + - name: 'default-receiver' + slack_configs: + - channel: '#alerts' + send_resolved: true + + - name: 'pagerduty-critical' + pagerduty_configs: + - service_key: 'xxx' + severity: critical + description: '{{ .CommonAnnotations.summary }}' + details: + firing: '{{ template "pagerduty.firing" . }}' + + - name: 'slack-high' + slack_configs: + - channel: '#alerts-high' + title: '{{ .CommonAnnotations.summary }}' + text: '{{ .CommonAnnotations.description }}' + actions: + - type: button + text: 'Runbook' + url: '{{ .CommonAnnotations.runbook_url }}' + - type: button + text: 'Dashboard' + url: '{{ .CommonAnnotations.dashboard_url }}' + + - name: 'platform-team' + slack_configs: + - channel: '#platform-alerts' + +time_intervals: + - name: business-hours + time_intervals: + - weekdays: ['monday:friday'] + times: + - start_time: '09:00' + end_time: '17:00' + +inhibit_rules: + - source_match: + severity: critical + target_match: + severity: high + equal: ['service'] +``` + +## PagerDuty Integration + +### Service Configuration + +```yaml +# Terraform example +resource "pagerduty_service" "myapp" { + name = "MyApp Production" + description = "Production application service" + escalation_policy = pagerduty_escalation_policy.default.id + alert_creation = "create_alerts_and_incidents" + auto_resolve_timeout = 14400 # 4 hours + acknowledgement_timeout = 600 # 10 minutes + + incident_urgency_rule { + type = "use_support_hours" + + during_support_hours { + type = "constant" + urgency = "high" + } + + outside_support_hours { + type = "constant" + urgency = "low" + } + } +} + +resource "pagerduty_escalation_policy" "default" { + name = "Default Escalation" + num_loops = 2 + + rule { + escalation_delay_in_minutes = 10 + target { + type = "schedule_reference" + id = pagerduty_schedule.primary.id + } + } + + rule { + escalation_delay_in_minutes = 15 + target { + type = "user_reference" + id = pagerduty_user.manager.id + } + } +} +``` + +### Schedule Configuration + +```yaml +resource "pagerduty_schedule" "primary" { + name = "Primary On-Call" + time_zone = "America/New_York" + + layer { + name = "Weekly Rotation" + start = "2024-01-01T00:00:00-05:00" + rotation_virtual_start = "2024-01-01T00:00:00-05:00" + rotation_turn_length_seconds = 604800 # 1 week + users = [for user in pagerduty_user.oncall : user.id] + } + + # Override layer for holidays + layer { + name = "Holiday Coverage" + start = "2024-01-01T00:00:00-05:00" + rotation_virtual_start = "2024-01-01T00:00:00-05:00" + rotation_turn_length_seconds = 86400 + users = [pagerduty_user.holiday_coverage.id] + + restriction { + type = "daily_restriction" + start_time_of_day = "00:00:00" + duration_seconds = 86400 + start_day_of_week = 0 # Sunday + } + } +} +``` + +## Grafana OnCall + +### Integration Setup + +```yaml +# docker-compose.yml addition +services: + oncall: + image: grafana/oncall + environment: + - SECRET_KEY=your-secret-key + - BASE_URL=http://oncall:8080 + - GRAFANA_API_URL=http://grafana:3000 + ports: + - "8080:8080" +``` + +### Escalation Chain + +```yaml +# Example escalation chain structure +escalation_chains: + - name: "Production Critical" + steps: + - step: 1 + type: notify + persons: + - "@oncall-primary" + wait_delay: 0 + + - step: 2 + type: notify + persons: + - "@oncall-secondary" + wait_delay: 5m + + - step: 3 + type: notify + persons: + - "@engineering-manager" + wait_delay: 10m + + - step: 4 + type: trigger_action + action: "escalate_to_incident_commander" + wait_delay: 15m +``` + +## Alert Templates + +### Slack Alert Template + +```go +{{ define "slack.title" }} +[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .CommonLabels.alertname }} +{{ end }} + +{{ define "slack.text" }} +{{ range .Alerts }} +*Alert:* {{ .Annotations.summary }} +*Severity:* {{ .Labels.severity }} +*Description:* {{ .Annotations.description }} +*Runbook:* {{ .Annotations.runbook_url }} +{{ end }} +{{ end }} +``` + +### PagerDuty Details Template + +```go +{{ define "pagerduty.firing" }} +{{ range .Alerts.Firing }} +Alert: {{ .Labels.alertname }} +Service: {{ .Labels.service }} +Instance: {{ .Labels.instance }} +Value: {{ .Annotations.value }} +Started: {{ .StartsAt.Format "2006-01-02 15:04:05" }} +{{ end }} +{{ end }} +``` + +## On-Call Best Practices + +### Rotation Guidelines + +```yaml +on_call_guidelines: + rotation_length: 1 week + handoff_time: "10:00 AM Monday" + + responsibilities: + - Monitor alerts during shift + - Respond within SLA (critical: 5min, high: 15min) + - Document incidents + - Handoff unresolved issues + + support: + - Secondary on-call for backup + - Clear escalation path + - Manager availability for major incidents + + wellness: + - Maximum 1 week on-call per month + - Comp time after high-alert periods + - No-interrupt recovery day after shift +``` + +### Runbook Template + +```markdown +# Alert: High Error Rate + +## Summary +Error rate has exceeded the threshold of 5% for the service. + +## Impact +Users may experience errors when accessing the application. + +## Investigation Steps +1. Check service logs: `kubectl logs -l app=myapp -n production` +2. Review recent deployments: `kubectl rollout history deployment/myapp` +3. Check database connectivity: `kubectl exec -it myapp -- nc -zv postgres 5432` +4. Review error traces in APM dashboard + +## Remediation +### If caused by recent deployment: +```bash +kubectl rollout undo deployment/myapp -n production +``` + +### If database related: +```bash +kubectl delete pod -l app=postgres -n production +``` + +## Escalation +If not resolved within 15 minutes, escalate to: +- Database team: @db-oncall +- Platform team: @platform-oncall +``` + +## Alert Fatigue Reduction + +### Strategies + +```yaml +fatigue_reduction: + aggregate_alerts: + - Group related alerts + - Use inhibit rules + - Implement alert correlation + + tune_thresholds: + - Base on SLOs, not arbitrary values + - Account for normal variance + - Use appropriate evaluation windows + + automate_responses: + - Auto-remediation for known issues + - Self-healing infrastructure + - Automated scaling + + regular_review: + - Weekly alert review + - Remove unused alerts + - Update thresholds based on data +``` + +## Common Issues + +### Issue: Alert Storm +**Problem**: Too many alerts firing simultaneously +**Solution**: Implement proper grouping and inhibition rules + +### Issue: Missed Alerts +**Problem**: Critical alerts not reaching on-call +**Solution**: Test escalation policies, verify contact methods + +### Issue: False Positives +**Problem**: Alerts firing without actual issues +**Solution**: Tune thresholds, increase evaluation windows + +## Best Practices + +- Define clear severity levels +- Every alert needs a runbook +- Test on-call notifications regularly +- Review and tune alerts weekly +- Implement proper escalation paths +- Use alert grouping and inhibition +- Track alert metrics (MTTR, frequency) +- Practice incident response regularly + +## Related Skills + +- [prometheus-grafana](../prometheus-grafana/) - Monitoring setup +- [incident-response](../../../security/operations/incident-response/) - Incident handling +- [runbook-automation](../../../compliance/continuity/runbook-automation/) - Runbook creation diff --git a/devops/observability/datadog/SKILL.md b/devops/observability/datadog/SKILL.md new file mode 100644 index 0000000..051cf00 --- /dev/null +++ b/devops/observability/datadog/SKILL.md @@ -0,0 +1,463 @@ +--- +name: datadog +description: Implement Datadog monitoring and APM for infrastructure and applications. Configure agents, create dashboards, set up alerts, and implement distributed tracing. Use when implementing enterprise monitoring, APM, or unified observability platforms. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# Datadog + +Monitor infrastructure and applications with Datadog's unified observability platform. + +## When to Use This Skill + +Use this skill when: +- Implementing enterprise-grade monitoring +- Setting up APM and distributed tracing +- Creating unified dashboards for infrastructure and apps +- Configuring intelligent alerting +- Monitoring cloud infrastructure (AWS, Azure, GCP) + +## Prerequisites + +- Datadog account and API key +- Agent installation access +- Application code access for APM + +## Agent Installation + +### Linux + +```bash +# Install agent +DD_API_KEY= DD_SITE="datadoghq.com" bash -c "$(curl -L https://s3.amazonaws.com/dd-agent/scripts/install_script_agent7.sh)" + +# Or via package manager +apt-get update && apt-get install datadog-agent + +# Configure API key +echo "api_key: YOUR_API_KEY" >> /etc/datadog-agent/datadog.yaml + +# Start agent +systemctl start datadog-agent +systemctl enable datadog-agent +``` + +### Docker + +```yaml +# docker-compose.yml +version: '3.8' + +services: + datadog-agent: + image: gcr.io/datadoghq/agent:7 + environment: + - DD_API_KEY=${DD_API_KEY} + - DD_SITE=datadoghq.com + - DD_LOGS_ENABLED=true + - DD_APM_ENABLED=true + - DD_PROCESS_AGENT_ENABLED=true + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + - /proc/:/host/proc/:ro + - /sys/fs/cgroup/:/host/sys/fs/cgroup:ro + ports: + - "8126:8126" # APM + - "8125:8125/udp" # DogStatsD +``` + +### Kubernetes + +```bash +# Using Helm +helm repo add datadog https://helm.datadoghq.com + +helm install datadog datadog/datadog \ + --set datadog.apiKey=${DD_API_KEY} \ + --set datadog.site=datadoghq.com \ + --set datadog.logs.enabled=true \ + --set datadog.apm.portEnabled=true \ + --set datadog.processAgent.enabled=true \ + --namespace datadog \ + --create-namespace +``` + +## Agent Configuration + +```yaml +# /etc/datadog-agent/datadog.yaml +api_key: YOUR_API_KEY +site: datadoghq.com + +# Hostname +hostname: myserver.example.com + +# Tags applied to all metrics +tags: + - env:production + - service:myapp + - team:platform + +# Log collection +logs_enabled: true + +# APM +apm_config: + enabled: true + apm_dd_url: https://trace.agent.datadoghq.com + +# Process monitoring +process_config: + enabled: true + +# Container monitoring +container_collect_all: true +docker_labels_as_tags: + app: service + environment: env +``` + +## Integration Configuration + +### MySQL + +```yaml +# /etc/datadog-agent/conf.d/mysql.d/conf.yaml +init_config: + +instances: + - host: localhost + port: 3306 + username: datadog + password: + tags: + - env:production + options: + replication: true + extra_status_metrics: true +``` + +### PostgreSQL + +```yaml +# /etc/datadog-agent/conf.d/postgres.d/conf.yaml +init_config: + +instances: + - host: localhost + port: 5432 + username: datadog + password: + dbname: mydb + collect_activity_metrics: true + collect_database_size_metrics: true +``` + +### NGINX + +```yaml +# /etc/datadog-agent/conf.d/nginx.d/conf.yaml +init_config: + +instances: + - nginx_status_url: http://localhost:80/nginx_status + tags: + - env:production +``` + +## Log Collection + +### File-Based Logs + +```yaml +# /etc/datadog-agent/conf.d/myapp.d/conf.yaml +logs: + - type: file + path: /var/log/myapp/*.log + service: myapp + source: python + sourcecategory: custom + tags: + - env:production + + - type: file + path: /var/log/nginx/access.log + service: nginx + source: nginx + log_processing_rules: + - type: exclude_at_match + name: exclude_healthchecks + pattern: health_check +``` + +### Docker Logs + +```yaml +# docker-compose.yml +services: + myapp: + labels: + com.datadoghq.ad.logs: '[{"source": "python", "service": "myapp"}]' +``` + +### Kubernetes Logs + +```yaml +# Pod annotation +apiVersion: v1 +kind: Pod +metadata: + annotations: + ad.datadoghq.com/myapp.logs: | + [{ + "source": "python", + "service": "myapp", + "log_processing_rules": [{ + "type": "multi_line", + "name": "python_tracebacks", + "pattern": "^Traceback" + }] + }] +``` + +## APM Configuration + +### Python + +```python +from ddtrace import patch_all, tracer + +# Automatic instrumentation +patch_all() + +# Configure tracer +tracer.configure( + hostname='localhost', + port=8126, + service='myapp', + env='production', + version='1.0.0' +) + +# Manual instrumentation +@tracer.wrap(service='myapp', resource='process_order') +def process_order(order_id): + with tracer.trace('validate_order') as span: + span.set_tag('order_id', order_id) + # Validation logic + + with tracer.trace('save_order'): + # Save logic + pass +``` + +```bash +# Install library +pip install ddtrace + +# Run with auto-instrumentation +ddtrace-run python app.py +``` + +### Node.js + +```javascript +const tracer = require('dd-trace').init({ + service: 'myapp', + env: 'production', + version: '1.0.0', + logInjection: true +}); + +// Manual instrumentation +const span = tracer.startSpan('custom_operation'); +span.setTag('user_id', userId); +// ... operation +span.finish(); +``` + +```bash +# Install library +npm install dd-trace + +# Run with auto-instrumentation +DD_TRACE_ENABLED=true node --require dd-trace/init app.js +``` + +### Go + +```go +import ( + "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer" +) + +func main() { + tracer.Start( + tracer.WithService("myapp"), + tracer.WithEnv("production"), + tracer.WithServiceVersion("1.0.0"), + ) + defer tracer.Stop() + + // Manual span + span, ctx := tracer.StartSpanFromContext(ctx, "process_request") + defer span.Finish() + span.SetTag("user_id", userID) +} +``` + +## Custom Metrics + +### DogStatsD + +```python +from datadog import DogStatsd + +statsd = DogStatsd(host='localhost', port=8125) + +# Counter +statsd.increment('myapp.orders.count', tags=['env:production']) + +# Gauge +statsd.gauge('myapp.queue.size', queue_size, tags=['queue:orders']) + +# Histogram +statsd.histogram('myapp.request.duration', response_time) + +# Distribution +statsd.distribution('myapp.response_time', duration, tags=['endpoint:/api/orders']) +``` + +### API Submission + +```python +from datadog_api_client import Configuration, ApiClient +from datadog_api_client.v2.api.metrics_api import MetricsApi +from datadog_api_client.v2.model.metric_payload import MetricPayload +from datadog_api_client.v2.model.metric_series import MetricSeries +from datadog_api_client.v2.model.metric_point import MetricPoint + +configuration = Configuration() +with ApiClient(configuration) as api_client: + api = MetricsApi(api_client) + + payload = MetricPayload( + series=[ + MetricSeries( + metric="custom.metric.name", + type=MetricSeries.GAUGE, + points=[MetricPoint(value=42.0, timestamp=int(time.time()))], + tags=["env:production"] + ) + ] + ) + api.submit_metrics(body=payload) +``` + +## Dashboards + +### Dashboard JSON + +```json +{ + "title": "Application Overview", + "widgets": [ + { + "definition": { + "type": "timeseries", + "title": "Request Rate", + "requests": [ + { + "q": "sum:trace.http.request.hits{service:myapp}.as_rate()", + "display_type": "line" + } + ] + } + }, + { + "definition": { + "type": "query_value", + "title": "Error Rate", + "requests": [ + { + "q": "sum:trace.http.request.errors{service:myapp}.as_rate() / sum:trace.http.request.hits{service:myapp}.as_rate() * 100" + } + ], + "precision": 2 + } + } + ] +} +``` + +## Monitors (Alerts) + +### Metric Monitor + +```json +{ + "name": "High Error Rate", + "type": "metric alert", + "query": "sum(last_5m):sum:trace.http.request.errors{service:myapp}.as_count() / sum:trace.http.request.hits{service:myapp}.as_count() > 0.05", + "message": "Error rate is {{value}}% for {{service.name}}. @slack-alerts", + "tags": ["service:myapp", "env:production"], + "options": { + "thresholds": { + "critical": 0.05, + "warning": 0.02 + }, + "notify_no_data": true, + "no_data_timeframe": 10 + } +} +``` + +### APM Monitor + +```json +{ + "name": "High Latency Alert", + "type": "trace-analytics alert", + "query": "trace-analytics(\"service:myapp @http.status_code:2*\").rollup(\"avg\", \"@duration\").last(\"5m\") > 2000000000", + "message": "Average latency is above 2 seconds. @pagerduty", + "options": { + "thresholds": { + "critical": 2000000000 + } + } +} +``` + +## Common Issues + +### Issue: Agent Not Reporting +**Problem**: No data appearing in Datadog +**Solution**: Check API key, verify agent status with `datadog-agent status` + +### Issue: Missing Traces +**Problem**: APM traces not appearing +**Solution**: Verify APM is enabled, check tracer configuration, verify port 8126 + +### Issue: High Cardinality Tags +**Problem**: Custom metrics getting dropped +**Solution**: Reduce unique tag values, use distributions instead of histograms + +## Best Practices + +- Use consistent service and environment tags +- Implement proper tag naming conventions +- Use unified service tagging (service, env, version) +- Set up service-level monitors +- Create dashboards per service +- Implement log correlation with traces +- Use distributions for latency metrics +- Configure proper alert escalation + +## Related Skills + +- [prometheus-grafana](../prometheus-grafana/) - Open source alternative +- [alerting-oncall](../alerting-oncall/) - Alert management +- [aws-vpc](../../../infrastructure/cloud-aws/aws-vpc/) - AWS monitoring diff --git a/devops/observability/datadog/references/datadog-integrations.md b/devops/observability/datadog/references/datadog-integrations.md new file mode 100644 index 0000000..a06d8c2 --- /dev/null +++ b/devops/observability/datadog/references/datadog-integrations.md @@ -0,0 +1,119 @@ +# Datadog Integration Reference + +## Agent Configuration + +```yaml +# /etc/datadog-agent/datadog.yaml +api_key: YOUR_API_KEY +site: datadoghq.com +hostname: my-host +tags: + - env:production + - team:platform + +logs_enabled: true +apm_config: + enabled: true +process_config: + enabled: true +``` + +## Docker Integration + +```yaml +# docker-compose.yml +datadog-agent: + image: gcr.io/datadoghq/agent:latest + environment: + - DD_API_KEY=${DD_API_KEY} + - DD_SITE=datadoghq.com + - DD_LOGS_ENABLED=true + - DD_APM_ENABLED=true + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + - /proc/:/host/proc/:ro + - /sys/fs/cgroup:/host/sys/fs/cgroup:ro +``` + +## Kubernetes Integration + +```yaml +# Datadog Agent Helm values +datadog: + apiKey: + site: datadoghq.com + + logs: + enabled: true + containerCollectAll: true + + apm: + portEnabled: true + + processAgent: + enabled: true + processCollection: true + +clusterAgent: + enabled: true + metricsProvider: + enabled: true +``` + +## Custom Metrics + +```python +from datadog import statsd + +# Counter +statsd.increment('page.views') + +# Gauge +statsd.gauge('users.online', 123) + +# Histogram +statsd.histogram('request.duration', 0.5) + +# Distribution +statsd.distribution('request.size', 1024) +``` + +## Log Integration + +```python +import logging +import json_log_formatter + +formatter = json_log_formatter.JSONFormatter() +handler = logging.StreamHandler() +handler.setFormatter(formatter) + +logger = logging.getLogger() +logger.addHandler(handler) +logger.setLevel(logging.INFO) + +logger.info('Request processed', extra={ + 'dd.trace_id': trace_id, + 'dd.span_id': span_id, + 'user_id': user_id +}) +``` + +## Monitors (Terraform) + +```hcl +resource "datadog_monitor" "cpu_high" { + name = "High CPU Usage" + type = "metric alert" + message = "CPU usage is high. @slack-alerts" + + query = "avg(last_5m):avg:system.cpu.user{*} by {host} > 80" + + monitor_thresholds { + critical = 80 + warning = 70 + } + + tags = ["env:production", "team:platform"] +} +``` diff --git a/devops/observability/elk-stack/SKILL.md b/devops/observability/elk-stack/SKILL.md new file mode 100644 index 0000000..095b14d --- /dev/null +++ b/devops/observability/elk-stack/SKILL.md @@ -0,0 +1,462 @@ +--- +name: elk-stack +description: Deploy and manage the ELK Stack (Elasticsearch, Logstash, Kibana) for log aggregation and analysis. Configure log pipelines, create visualizations, and implement log-based monitoring. Use when centralizing logs, implementing search functionality, or building log analytics platforms. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# ELK Stack + +Centralize and analyze logs with Elasticsearch, Logstash, and Kibana. + +## When to Use This Skill + +Use this skill when: +- Centralizing logs from multiple sources +- Building log search and analytics platforms +- Creating log-based dashboards and alerts +- Implementing full-text search for logs +- Processing and transforming log data + +## Prerequisites + +- Docker or server infrastructure +- Sufficient disk space for log storage +- Network access from log sources + +## Docker Deployment + +```yaml +# docker-compose.yml +version: '3.8' + +services: + elasticsearch: + image: docker.elastic.co/elasticsearch/elasticsearch:8.11.0 + environment: + - discovery.type=single-node + - xpack.security.enabled=false + - "ES_JAVA_OPTS=-Xms1g -Xmx1g" + ports: + - "9200:9200" + volumes: + - elasticsearch-data:/usr/share/elasticsearch/data + + logstash: + image: docker.elastic.co/logstash/logstash:8.11.0 + volumes: + - ./logstash/pipeline:/usr/share/logstash/pipeline + - ./logstash/config:/usr/share/logstash/config + ports: + - "5044:5044" + - "5000:5000" + depends_on: + - elasticsearch + + kibana: + image: docker.elastic.co/kibana/kibana:8.11.0 + ports: + - "5601:5601" + environment: + - ELASTICSEARCH_HOSTS=http://elasticsearch:9200 + depends_on: + - elasticsearch + + filebeat: + image: docker.elastic.co/beats/filebeat:8.11.0 + user: root + volumes: + - ./filebeat/filebeat.yml:/usr/share/filebeat/filebeat.yml:ro + - /var/lib/docker/containers:/var/lib/docker/containers:ro + - /var/run/docker.sock:/var/run/docker.sock:ro + depends_on: + - logstash + +volumes: + elasticsearch-data: +``` + +## Elasticsearch Configuration + +### Index Templates + +```json +PUT _index_template/logs-template +{ + "index_patterns": ["logs-*"], + "template": { + "settings": { + "number_of_shards": 1, + "number_of_replicas": 1, + "index.lifecycle.name": "logs-policy" + }, + "mappings": { + "properties": { + "@timestamp": { "type": "date" }, + "message": { "type": "text" }, + "level": { "type": "keyword" }, + "service": { "type": "keyword" }, + "host": { "type": "keyword" }, + "trace_id": { "type": "keyword" } + } + } + } +} +``` + +### Index Lifecycle Management + +```json +PUT _ilm/policy/logs-policy +{ + "policy": { + "phases": { + "hot": { + "min_age": "0ms", + "actions": { + "rollover": { + "max_size": "50GB", + "max_age": "1d" + } + } + }, + "warm": { + "min_age": "7d", + "actions": { + "shrink": { "number_of_shards": 1 }, + "forcemerge": { "max_num_segments": 1 } + } + }, + "cold": { + "min_age": "30d", + "actions": { + "freeze": {} + } + }, + "delete": { + "min_age": "90d", + "actions": { + "delete": {} + } + } + } + } +} +``` + +## Logstash Pipeline + +### Basic Pipeline + +```ruby +# logstash/pipeline/main.conf +input { + beats { + port => 5044 + } + + tcp { + port => 5000 + codec => json_lines + } +} + +filter { + # Parse JSON logs + if [message] =~ /^\{/ { + json { + source => "message" + } + } + + # Parse timestamp + date { + match => ["timestamp", "ISO8601", "yyyy-MM-dd HH:mm:ss"] + target => "@timestamp" + } + + # Add environment tag + mutate { + add_field => { "environment" => "production" } + } + + # Grok pattern for nginx logs + if [type] == "nginx" { + grok { + match => { + "message" => '%{IPORHOST:client_ip} - %{USER:user} \[%{HTTPDATE:timestamp}\] "%{WORD:method} %{URIPATHPARAM:request} HTTP/%{NUMBER:http_version}" %{NUMBER:status} %{NUMBER:bytes}' + } + } + } +} + +output { + elasticsearch { + hosts => ["elasticsearch:9200"] + index => "logs-%{+YYYY.MM.dd}" + } +} +``` + +### Advanced Filtering + +```ruby +filter { + # Parse application logs + grok { + match => { + "message" => "%{TIMESTAMP_ISO8601:timestamp} %{LOGLEVEL:level} \[%{DATA:service}\] %{GREEDYDATA:log_message}" + } + } + + # Extract trace ID from message + if [log_message] =~ /trace_id=/ { + grok { + match => { "log_message" => "trace_id=%{UUID:trace_id}" } + } + } + + # GeoIP lookup + if [client_ip] { + geoip { + source => "client_ip" + target => "geoip" + } + } + + # Drop debug logs in production + if [level] == "DEBUG" and [environment] == "production" { + drop {} + } + + # Enrich with lookup + translate { + field => "status" + destination => "status_description" + dictionary => { + "200" => "OK" + "404" => "Not Found" + "500" => "Internal Server Error" + } + } +} +``` + +## Filebeat Configuration + +```yaml +# filebeat/filebeat.yml +filebeat.inputs: + - type: container + paths: + - '/var/lib/docker/containers/*/*.log' + processors: + - add_docker_metadata: + host: "unix:///var/run/docker.sock" + + - type: log + enabled: true + paths: + - /var/log/nginx/*.log + tags: ["nginx"] + fields: + type: nginx + +output.logstash: + hosts: ["logstash:5044"] + +logging.level: info +logging.to_files: true +logging.files: + path: /var/log/filebeat + name: filebeat + keepfiles: 7 +``` + +## Elasticsearch Queries + +### Basic Queries + +```json +// Search all logs +GET logs-*/_search +{ + "query": { + "match_all": {} + } +} + +// Search by keyword +GET logs-*/_search +{ + "query": { + "match": { + "message": "error" + } + } +} + +// Filter by field +GET logs-*/_search +{ + "query": { + "bool": { + "must": [ + { "match": { "level": "ERROR" } }, + { "range": { "@timestamp": { "gte": "now-1h" } } } + ], + "filter": [ + { "term": { "service": "api-gateway" } } + ] + } + } +} +``` + +### Aggregations + +```json +// Count by log level +GET logs-*/_search +{ + "size": 0, + "aggs": { + "log_levels": { + "terms": { "field": "level" } + } + } +} + +// Error rate over time +GET logs-*/_search +{ + "size": 0, + "aggs": { + "errors_over_time": { + "date_histogram": { + "field": "@timestamp", + "fixed_interval": "5m" + }, + "aggs": { + "error_count": { + "filter": { "term": { "level": "ERROR" } } + } + } + } + } +} +``` + +## Kibana Setup + +### Index Patterns + +1. Go to Stack Management → Index Patterns +2. Create pattern: `logs-*` +3. Set time field: `@timestamp` + +### Saved Searches + +Create saved searches for common queries: +- `level:ERROR` - All errors +- `service:api-gateway AND level:ERROR` - API gateway errors +- `response_time:>1000` - Slow requests + +### Visualizations + +Common visualization types: +- **Line Chart**: Error rate over time +- **Pie Chart**: Distribution by log level +- **Data Table**: Top error messages +- **Metric**: Total error count + +### Dashboard Example + +Create dashboard with: +1. Total log count (Metric) +2. Error rate trend (Line chart) +3. Logs by service (Pie chart) +4. Recent errors (Data table) +5. Log stream (Discover panel) + +## Alerting + +### Watcher (X-Pack) + +```json +PUT _watcher/watch/error_alert +{ + "trigger": { + "schedule": { "interval": "5m" } + }, + "input": { + "search": { + "request": { + "indices": ["logs-*"], + "body": { + "query": { + "bool": { + "must": [ + { "match": { "level": "ERROR" } }, + { "range": { "@timestamp": { "gte": "now-5m" } } } + ] + } + } + } + } + } + }, + "condition": { + "compare": { "ctx.payload.hits.total.value": { "gt": 100 } } + }, + "actions": { + "notify_slack": { + "webhook": { + "scheme": "https", + "host": "hooks.slack.com", + "port": 443, + "method": "post", + "path": "/services/xxx", + "body": "{\"text\": \"High error rate detected: {{ctx.payload.hits.total.value}} errors in last 5 minutes\"}" + } + } + } +} +``` + +## Common Issues + +### Issue: High Disk Usage +**Problem**: Elasticsearch consuming too much disk +**Solution**: Implement ILM policies, reduce retention + +### Issue: Slow Searches +**Problem**: Queries taking too long +**Solution**: Optimize index settings, add more shards, use filters + +### Issue: Log Parsing Failures +**Problem**: Logs not parsed correctly +**Solution**: Test grok patterns, check for log format changes + +### Issue: Memory Pressure +**Problem**: Elasticsearch OOM errors +**Solution**: Increase heap size (max 50% of RAM), limit field data + +## Best Practices + +- Implement index lifecycle management +- Use index templates for consistent mappings +- Parse logs at ingestion time +- Limit stored fields to reduce storage +- Use data streams for time-series data +- Monitor cluster health +- Implement proper security (X-Pack) +- Regular index maintenance + +## Related Skills + +- [loki-logging](../loki-logging/) - Alternative logging stack +- [prometheus-grafana](../prometheus-grafana/) - Metrics monitoring +- [audit-logging](../../../compliance/auditing/audit-logging/) - Compliance logging diff --git a/devops/observability/elk-stack/references/elasticsearch-queries.md b/devops/observability/elk-stack/references/elasticsearch-queries.md new file mode 100644 index 0000000..990d5ad --- /dev/null +++ b/devops/observability/elk-stack/references/elasticsearch-queries.md @@ -0,0 +1,139 @@ +# Elasticsearch Query Reference + +## Basic Queries + +```json +// Match all +GET /logs/_search +{ + "query": { "match_all": {} } +} + +// Match query +GET /logs/_search +{ + "query": { + "match": { "message": "error" } + } +} + +// Term query (exact match) +GET /logs/_search +{ + "query": { + "term": { "status": "500" } + } +} +``` + +## Boolean Queries + +```json +GET /logs/_search +{ + "query": { + "bool": { + "must": [ + { "match": { "service": "api" } } + ], + "filter": [ + { "range": { "@timestamp": { "gte": "now-1h" } } } + ], + "should": [ + { "match": { "level": "error" } } + ], + "must_not": [ + { "term": { "environment": "test" } } + ] + } + } +} +``` + +## Aggregations + +```json +// Terms aggregation +GET /logs/_search +{ + "size": 0, + "aggs": { + "by_status": { + "terms": { "field": "status.keyword" } + } + } +} + +// Date histogram +GET /logs/_search +{ + "size": 0, + "aggs": { + "over_time": { + "date_histogram": { + "field": "@timestamp", + "fixed_interval": "1h" + } + } + } +} + +// Nested aggregations +GET /logs/_search +{ + "size": 0, + "aggs": { + "by_service": { + "terms": { "field": "service.keyword" }, + "aggs": { + "error_count": { + "filter": { "term": { "level": "error" } } + } + } + } + } +} +``` + +## Index Management + +```bash +# Create index +PUT /logs-2024 +{ + "settings": { + "number_of_shards": 3, + "number_of_replicas": 1 + } +} + +# Index template +PUT /_index_template/logs +{ + "index_patterns": ["logs-*"], + "template": { + "settings": { + "number_of_shards": 3 + }, + "mappings": { + "properties": { + "@timestamp": { "type": "date" }, + "message": { "type": "text" }, + "level": { "type": "keyword" } + } + } + } +} + +# ILM policy +PUT /_ilm/policy/logs-policy +{ + "policy": { + "phases": { + "hot": { "actions": { "rollover": { "max_size": "50GB" } } }, + "warm": { "min_age": "7d", "actions": { "shrink": { "number_of_shards": 1 } } }, + "delete": { "min_age": "30d", "actions": { "delete": {} } } + } + } +} +``` diff --git a/devops/observability/loki-logging/SKILL.md b/devops/observability/loki-logging/SKILL.md new file mode 100644 index 0000000..fb5698a --- /dev/null +++ b/devops/observability/loki-logging/SKILL.md @@ -0,0 +1,455 @@ +--- +name: loki-logging +description: Configure Grafana Loki for log aggregation and analysis. Set up Promtail for log collection, write LogQL queries, and integrate with Grafana for visualization. Use when implementing lightweight log aggregation, especially in Kubernetes environments. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# Grafana Loki + +Aggregate and query logs with Grafana Loki, the Prometheus-inspired logging system. + +## When to Use This Skill + +Use this skill when: +- Implementing cost-effective log aggregation +- Building logging for Kubernetes environments +- Integrating logs with Grafana dashboards +- Querying logs with label-based filtering +- Preferring lighter-weight alternative to ELK + +## Prerequisites + +- Docker or Kubernetes +- Grafana for visualization +- Promtail or other log shipper + +## Architecture Overview + +``` +┌─────────────┐ ┌──────────┐ ┌──────────┐ +│ Application │────▶│ Promtail │────▶│ Loki │ +└─────────────┘ └──────────┘ └──────────┘ + │ + ▼ + ┌──────────┐ + │ Grafana │ + └──────────┘ +``` + +## Docker Deployment + +```yaml +# docker-compose.yml +version: '3.8' + +services: + loki: + image: grafana/loki:2.9.0 + ports: + - "3100:3100" + volumes: + - ./loki-config.yaml:/etc/loki/local-config.yaml + - loki-data:/loki + command: -config.file=/etc/loki/local-config.yaml + + promtail: + image: grafana/promtail:2.9.0 + volumes: + - ./promtail-config.yaml:/etc/promtail/config.yaml + - /var/log:/var/log:ro + - /var/lib/docker/containers:/var/lib/docker/containers:ro + command: -config.file=/etc/promtail/config.yaml + + grafana: + image: grafana/grafana:10.2.0 + ports: + - "3000:3000" + volumes: + - grafana-data:/var/lib/grafana + - ./grafana/provisioning:/etc/grafana/provisioning + environment: + - GF_AUTH_ANONYMOUS_ENABLED=true + - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin + +volumes: + loki-data: + grafana-data: +``` + +## Loki Configuration + +```yaml +# loki-config.yaml +auth_enabled: false + +server: + http_listen_port: 3100 + +common: + path_prefix: /loki + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory + +schema_config: + configs: + - from: 2020-10-24 + store: boltdb-shipper + object_store: filesystem + schema: v11 + index: + prefix: index_ + period: 24h + +storage_config: + boltdb_shipper: + active_index_directory: /loki/index + cache_location: /loki/cache + shared_store: filesystem + +limits_config: + reject_old_samples: true + reject_old_samples_max_age: 168h + max_query_series: 5000 + max_query_parallelism: 2 + +chunk_store_config: + max_look_back_period: 168h + +table_manager: + retention_deletes_enabled: true + retention_period: 168h +``` + +## Promtail Configuration + +```yaml +# promtail-config.yaml +server: + http_listen_port: 9080 + grpc_listen_port: 0 + +positions: + filename: /tmp/positions.yaml + +clients: + - url: http://loki:3100/loki/api/v1/push + +scrape_configs: + # System logs + - job_name: system + static_configs: + - targets: + - localhost + labels: + job: varlogs + __path__: /var/log/*.log + + # Docker container logs + - job_name: docker + docker_sd_configs: + - host: unix:///var/run/docker.sock + refresh_interval: 5s + relabel_configs: + - source_labels: ['__meta_docker_container_name'] + regex: '/(.*)' + target_label: 'container' + - source_labels: ['__meta_docker_container_log_stream'] + target_label: 'stream' + + # Application logs with parsing + - job_name: application + static_configs: + - targets: + - localhost + labels: + job: application + __path__: /var/log/app/*.log + pipeline_stages: + - json: + expressions: + level: level + message: message + timestamp: timestamp + - labels: + level: + - timestamp: + source: timestamp + format: RFC3339 +``` + +## Kubernetes Deployment + +```bash +# Using Helm +helm repo add grafana https://grafana.github.io/helm-charts +helm install loki grafana/loki-stack \ + --namespace monitoring \ + --create-namespace \ + --set grafana.enabled=true \ + --set promtail.enabled=true +``` + +### Promtail DaemonSet + +```yaml +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: promtail + namespace: monitoring +spec: + selector: + matchLabels: + app: promtail + template: + metadata: + labels: + app: promtail + spec: + containers: + - name: promtail + image: grafana/promtail:2.9.0 + args: + - -config.file=/etc/promtail/promtail.yaml + volumeMounts: + - name: config + mountPath: /etc/promtail + - name: varlog + mountPath: /var/log + - name: varlibdockercontainers + mountPath: /var/lib/docker/containers + readOnly: true + volumes: + - name: config + configMap: + name: promtail-config + - name: varlog + hostPath: + path: /var/log + - name: varlibdockercontainers + hostPath: + path: /var/lib/docker/containers +``` + +## LogQL Queries + +### Basic Queries + +```logql +# All logs from a job +{job="application"} + +# Filter by label +{job="application", level="error"} + +# Multiple labels +{namespace="production", container="api"} + +# Regex match +{job=~"app.*"} +``` + +### Log Pipeline + +```logql +# Filter by content +{job="application"} |= "error" + +# Exclude content +{job="application"} != "debug" + +# Regex filter +{job="application"} |~ "user_id=[0-9]+" + +# JSON parsing +{job="application"} | json | level="error" + +# Line format +{job="application"} | json | line_format "{{.level}}: {{.message}}" +``` + +### Metric Queries + +```logql +# Count logs per second +count_over_time({job="application"}[5m]) + +# Rate of errors +rate({job="application", level="error"}[5m]) + +# Sum by label +sum by (level) (count_over_time({job="application"}[5m])) + +# Top services by error count +topk(5, sum by (service) (count_over_time({level="error"}[1h]))) +``` + +### Aggregations + +```logql +# Average log line length +avg_over_time({job="application"} | unwrap line_length [5m]) + +# Percentile of numeric field +quantile_over_time(0.95, {job="application"} | json | unwrap response_time [5m]) + +# Error percentage +sum(rate({job="application", level="error"}[5m])) +/ +sum(rate({job="application"}[5m])) * 100 +``` + +## Pipeline Stages + +```yaml +# promtail-config.yaml +pipeline_stages: + # Parse JSON logs + - json: + expressions: + level: level + message: msg + trace_id: trace_id + + # Extract with regex + - regex: + expression: 'user_id=(?P\d+)' + + # Add labels from parsed fields + - labels: + level: + user_id: + + # Modify timestamp + - timestamp: + source: timestamp + format: '2006-01-02T15:04:05.000Z' + + # Filter logs + - match: + selector: '{level="debug"}' + action: drop + + # Add static labels + - static_labels: + environment: production + + # Modify log line + - template: + source: message + template: '{{ ToUpper .Value }}' +``` + +## Grafana Integration + +### Data Source Configuration + +```yaml +# grafana/provisioning/datasources/loki.yaml +apiVersion: 1 + +datasources: + - name: Loki + type: loki + access: proxy + url: http://loki:3100 + isDefault: false + jsonData: + maxLines: 1000 +``` + +### Dashboard Panel + +```json +{ + "title": "Application Logs", + "type": "logs", + "datasource": "Loki", + "targets": [ + { + "expr": "{job=\"application\"} | json", + "refId": "A" + } + ], + "options": { + "showTime": true, + "showLabels": true, + "wrapLogMessage": true + } +} +``` + +## Recording Rules + +```yaml +# loki-rules.yaml +groups: + - name: error_rates + interval: 1m + rules: + - record: job:log_errors:rate5m + expr: | + sum by (job) (rate({level="error"}[5m])) +``` + +## Alerting + +```yaml +# loki-alerts.yaml +groups: + - name: log_alerts + rules: + - alert: HighErrorRate + expr: | + sum(rate({level="error"}[5m])) > 10 + for: 5m + labels: + severity: critical + annotations: + summary: "High error rate in logs" + description: "Error rate is {{ $value }} errors/second" +``` + +## Common Issues + +### Issue: High Memory Usage +**Problem**: Loki consuming too much memory +**Solution**: Reduce max_query_series, limit query time range + +### Issue: Logs Not Appearing +**Problem**: Promtail not shipping logs +**Solution**: Check positions file, verify file paths, check label configuration + +### Issue: Query Timeout +**Problem**: LogQL queries timing out +**Solution**: Add more specific label filters, reduce time range + +### Issue: Ingestion Rate Limit +**Problem**: Logs being dropped +**Solution**: Increase per_stream_rate_limit in limits_config + +## Best Practices + +- Use meaningful labels (avoid high cardinality) +- Filter by labels before log content +- Parse logs at collection time with Promtail +- Set appropriate retention periods +- Use recording rules for common queries +- Implement proper multitenancy for large deployments +- Monitor Loki's own metrics +- Use chunk caching for better performance + +## Related Skills + +- [prometheus-grafana](../prometheus-grafana/) - Metrics monitoring +- [elk-stack](../elk-stack/) - Alternative logging +- [alerting-oncall](../alerting-oncall/) - Alert management diff --git a/devops/observability/new-relic/SKILL.md b/devops/observability/new-relic/SKILL.md new file mode 100644 index 0000000..b41ed89 --- /dev/null +++ b/devops/observability/new-relic/SKILL.md @@ -0,0 +1,447 @@ +--- +name: new-relic +description: Configure New Relic observability platform for infrastructure and application monitoring. Set up APM agents, create dashboards, configure alerts, and implement distributed tracing. Use when implementing full-stack observability with New Relic One. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# New Relic + +Monitor applications and infrastructure with New Relic's observability platform. + +## When to Use This Skill + +Use this skill when: +- Implementing full-stack observability +- Setting up APM for applications +- Monitoring infrastructure health +- Creating custom dashboards and alerts +- Implementing distributed tracing + +## Prerequisites + +- New Relic account and license key +- Application access for APM agents +- Infrastructure access for host agents + +## Infrastructure Agent + +### Linux Installation + +```bash +# Add repository and install +curl -Ls https://download.newrelic.com/install/newrelic-cli/scripts/install.sh | bash + +# Configure license key +sudo NEW_RELIC_API_KEY= NEW_RELIC_ACCOUNT_ID= /usr/local/bin/newrelic install + +# Or manual configuration +echo "license_key: YOUR_LICENSE_KEY" | sudo tee -a /etc/newrelic-infra.yml +sudo systemctl start newrelic-infra +``` + +### Docker + +```yaml +# docker-compose.yml +version: '3.8' + +services: + newrelic-infra: + image: newrelic/infrastructure:latest + cap_add: + - SYS_PTRACE + privileged: true + pid: "host" + network_mode: "host" + environment: + - NRIA_LICENSE_KEY=${NEW_RELIC_LICENSE_KEY} + - NRIA_DISPLAY_NAME=docker-host + volumes: + - /:/host:ro + - /var/run/docker.sock:/var/run/docker.sock +``` + +### Kubernetes + +```bash +# Using Helm +helm repo add newrelic https://helm-charts.newrelic.com + +helm install newrelic-bundle newrelic/nri-bundle \ + --namespace newrelic \ + --create-namespace \ + --set global.licenseKey=${NEW_RELIC_LICENSE_KEY} \ + --set global.cluster=my-cluster \ + --set newrelic-infrastructure.privileged=true \ + --set ksm.enabled=true \ + --set kubeEvents.enabled=true \ + --set logging.enabled=true +``` + +## APM Agents + +### Node.js + +```javascript +// At the very start of your application +require('newrelic'); + +// newrelic.js configuration +exports.config = { + app_name: ['My Application'], + license_key: process.env.NEW_RELIC_LICENSE_KEY, + distributed_tracing: { + enabled: true + }, + logging: { + level: 'info' + }, + error_collector: { + enabled: true, + ignore_status_codes: [404] + }, + transaction_tracer: { + enabled: true, + transaction_threshold: 'apdex_f', + record_sql: 'obfuscated' + } +}; +``` + +```bash +# Install agent +npm install newrelic + +# Run application +NEW_RELIC_LICENSE_KEY=xxx node -r newrelic app.js +``` + +### Python + +```python +# newrelic.ini +[newrelic] +license_key = YOUR_LICENSE_KEY +app_name = My Application +distributed_tracing.enabled = true +transaction_tracer.enabled = true +error_collector.enabled = true +browser_monitoring.auto_instrument = true +``` + +```bash +# Install agent +pip install newrelic + +# Generate config file +newrelic-admin generate-config YOUR_LICENSE_KEY newrelic.ini + +# Run application +NEW_RELIC_CONFIG_FILE=newrelic.ini newrelic-admin run-program python app.py + +# Or with gunicorn +NEW_RELIC_CONFIG_FILE=newrelic.ini newrelic-admin run-program gunicorn app:app +``` + +### Java + +```bash +# Download agent +curl -O https://download.newrelic.com/newrelic/java-agent/newrelic-agent/current/newrelic-java.zip +unzip newrelic-java.zip + +# Configure newrelic.yml +# license_key: YOUR_LICENSE_KEY +# app_name: My Application + +# Run with agent +java -javaagent:/path/to/newrelic.jar -jar myapp.jar +``` + +### Go + +```go +package main + +import ( + "github.com/newrelic/go-agent/v3/newrelic" + "net/http" +) + +func main() { + app, err := newrelic.NewApplication( + newrelic.ConfigAppName("My Application"), + newrelic.ConfigLicense("YOUR_LICENSE_KEY"), + newrelic.ConfigDistributedTracerEnabled(true), + ) + if err != nil { + panic(err) + } + + http.HandleFunc(newrelic.WrapHandleFunc(app, "/", indexHandler)) + http.ListenAndServe(":8080", nil) +} + +func indexHandler(w http.ResponseWriter, r *http.Request) { + txn := newrelic.FromContext(r.Context()) + txn.AddAttribute("user_id", "12345") + w.Write([]byte("Hello, World!")) +} +``` + +## Custom Instrumentation + +### Custom Events + +```python +import newrelic.agent + +# Record custom event +newrelic.agent.record_custom_event('OrderPlaced', { + 'order_id': '12345', + 'amount': 99.99, + 'customer_id': 'cust_001' +}) +``` + +### Custom Metrics + +```python +import newrelic.agent + +# Record custom metric +newrelic.agent.record_custom_metric('Custom/OrderValue', 99.99) + +# With attributes +newrelic.agent.record_custom_metric('Custom/ProcessingTime', + processing_time, + {'unit': 'milliseconds'} +) +``` + +### Custom Spans + +```python +import newrelic.agent + +@newrelic.agent.function_trace(name='process_payment') +def process_payment(order_id, amount): + # This creates a custom span in the trace + pass + +# Manual span creation +with newrelic.agent.FunctionTrace(name='custom_operation'): + # Traced code + pass +``` + +## NRQL Queries + +### Basic Queries + +```sql +-- Transaction throughput +SELECT rate(count(*), 1 minute) FROM Transaction +WHERE appName = 'My Application' +SINCE 1 hour ago + +-- Average response time +SELECT average(duration) FROM Transaction +WHERE appName = 'My Application' +SINCE 1 hour ago + +-- Error rate +SELECT percentage(count(*), WHERE error IS true) FROM Transaction +WHERE appName = 'My Application' +SINCE 1 hour ago + +-- Apdex score +SELECT apdex(duration, t: 0.5) FROM Transaction +WHERE appName = 'My Application' +SINCE 1 hour ago +``` + +### Advanced Queries + +```sql +-- Slowest transactions +SELECT average(duration) FROM Transaction +WHERE appName = 'My Application' +FACET name +SINCE 1 hour ago +ORDER BY average(duration) DESC +LIMIT 10 + +-- Error breakdown +SELECT count(*) FROM TransactionError +WHERE appName = 'My Application' +FACET error.class +SINCE 1 hour ago + +-- Percentile response times +SELECT percentile(duration, 50, 90, 95, 99) FROM Transaction +WHERE appName = 'My Application' +SINCE 1 hour ago TIMESERIES + +-- Custom event analysis +SELECT average(amount), count(*) FROM OrderPlaced +FACET customer_id +SINCE 1 day ago +``` + +## Dashboards + +### Dashboard JSON + +```json +{ + "name": "Application Dashboard", + "pages": [ + { + "name": "Overview", + "widgets": [ + { + "title": "Throughput", + "visualization": {"id": "viz.line"}, + "configuration": { + "nrqlQueries": [ + { + "accountId": 12345, + "query": "SELECT rate(count(*), 1 minute) FROM Transaction WHERE appName = 'My Application' SINCE 1 hour ago TIMESERIES" + } + ] + } + }, + { + "title": "Error Rate", + "visualization": {"id": "viz.billboard"}, + "configuration": { + "nrqlQueries": [ + { + "accountId": 12345, + "query": "SELECT percentage(count(*), WHERE error IS true) FROM Transaction WHERE appName = 'My Application' SINCE 1 hour ago" + } + ] + } + } + ] + } + ] +} +``` + +## Alerts + +### Alert Condition (NRQL) + +```json +{ + "name": "High Error Rate", + "type": "static", + "nrql": { + "query": "SELECT percentage(count(*), WHERE error IS true) FROM Transaction WHERE appName = 'My Application'" + }, + "valueFunction": "single_value", + "terms": [ + { + "threshold": 5, + "thresholdOccurrences": "all", + "thresholdDuration": 300, + "operator": "above", + "priority": "critical" + }, + { + "threshold": 2, + "thresholdOccurrences": "all", + "thresholdDuration": 300, + "operator": "above", + "priority": "warning" + } + ] +} +``` + +### Alert Policy + +```json +{ + "name": "Application Alerts", + "incident_preference": "PER_CONDITION_AND_TARGET", + "conditions": [ + { + "name": "High Response Time", + "type": "apm_app_metric", + "entities": ["My Application"], + "metric": "response_time_web", + "condition_scope": "application", + "terms": [ + { + "duration": "5", + "operator": "above", + "threshold": "1", + "priority": "critical" + } + ] + } + ] +} +``` + +## Logs in Context + +### Python Configuration + +```python +# newrelic.ini +[newrelic] +application_logging.enabled = true +application_logging.forwarding.enabled = true +application_logging.metrics.enabled = true +application_logging.local_decorating.enabled = true +``` + +### Log Forwarding + +```yaml +# newrelic-infra.yml +log: + - name: application-logs + file: /var/log/myapp/*.log + attributes: + service: myapp + environment: production +``` + +## Common Issues + +### Issue: No Data Appearing +**Problem**: Agent not reporting to New Relic +**Solution**: Verify license key, check network connectivity, review agent logs + +### Issue: Missing Transactions +**Problem**: Some transactions not captured +**Solution**: Check instrumentation coverage, verify framework support + +### Issue: High Overhead +**Problem**: APM agent impacting performance +**Solution**: Adjust sampling rate, disable unnecessary features + +## Best Practices + +- Use meaningful application names +- Implement distributed tracing across services +- Set up service maps for dependency visualization +- Configure appropriate alert thresholds +- Use custom attributes for business context +- Implement logs in context for correlation +- Set up workloads for service grouping +- Regular review of unused dashboards and alerts + +## Related Skills + +- [datadog](../datadog/) - Alternative monitoring platform +- [prometheus-grafana](../prometheus-grafana/) - Open source monitoring +- [alerting-oncall](../alerting-oncall/) - Alert management diff --git a/devops/observability/prometheus-grafana/SKILL.md b/devops/observability/prometheus-grafana/SKILL.md new file mode 100644 index 0000000..311e02e --- /dev/null +++ b/devops/observability/prometheus-grafana/SKILL.md @@ -0,0 +1,445 @@ +--- +name: prometheus-grafana +description: Set up metrics collection and visualization with Prometheus and Grafana. Configure scrape targets, create PromQL queries, build dashboards, and implement alerting. Use when implementing monitoring, metrics collection, or visualization for applications and infrastructure. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# Prometheus & Grafana + +Collect metrics and visualize system performance with the Prometheus-Grafana stack. + +## When to Use This Skill + +Use this skill when: +- Setting up metrics collection infrastructure +- Creating monitoring dashboards +- Writing PromQL queries for analysis +- Configuring alerting rules +- Monitoring Kubernetes clusters + +## Prerequisites + +- Docker or Kubernetes for deployment +- Network access to monitored targets +- Basic understanding of metrics concepts + +## Prometheus Setup + +### Docker Deployment + +```yaml +# docker-compose.yml +version: '3.8' + +services: + prometheus: + image: prom/prometheus:v2.48.0 + ports: + - "9090:9090" + volumes: + - ./prometheus.yml:/etc/prometheus/prometheus.yml + - ./rules:/etc/prometheus/rules + - prometheus-data:/prometheus + command: + - '--config.file=/etc/prometheus/prometheus.yml' + - '--storage.tsdb.path=/prometheus' + - '--storage.tsdb.retention.time=15d' + + grafana: + image: grafana/grafana:10.2.0 + ports: + - "3000:3000" + volumes: + - grafana-data:/var/lib/grafana + environment: + - GF_SECURITY_ADMIN_PASSWORD=admin + +volumes: + prometheus-data: + grafana-data: +``` + +### Configuration + +```yaml +# prometheus.yml +global: + scrape_interval: 15s + evaluation_interval: 15s + +alerting: + alertmanagers: + - static_configs: + - targets: + - alertmanager:9093 + +rule_files: + - /etc/prometheus/rules/*.yml + +scrape_configs: + - job_name: 'prometheus' + static_configs: + - targets: ['localhost:9090'] + + - job_name: 'node' + static_configs: + - targets: + - 'node-exporter:9100' + + - job_name: 'applications' + static_configs: + - targets: + - 'app1:8080' + - 'app2:8080' + metrics_path: /metrics +``` + +## Kubernetes Deployment + +### Using Helm + +```bash +# Add Prometheus community Helm repo +helm repo add prometheus-community https://prometheus-community.github.io/helm-charts + +# Install kube-prometheus-stack +helm install prometheus prometheus-community/kube-prometheus-stack \ + --namespace monitoring \ + --create-namespace \ + --set grafana.adminPassword=admin +``` + +### ServiceMonitor + +```yaml +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: myapp + namespace: monitoring +spec: + selector: + matchLabels: + app: myapp + endpoints: + - port: metrics + interval: 30s + path: /metrics + namespaceSelector: + matchNames: + - default +``` + +## PromQL Queries + +### Basic Queries + +```promql +# Current CPU usage +node_cpu_seconds_total{mode="idle"} + +# Rate of HTTP requests per second +rate(http_requests_total[5m]) + +# Average response time +avg(http_request_duration_seconds_sum / http_request_duration_seconds_count) + +# Memory usage percentage +(1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100 +``` + +### Aggregations + +```promql +# Sum requests by status code +sum by (status_code) (rate(http_requests_total[5m])) + +# Average CPU by instance +avg by (instance) (rate(node_cpu_seconds_total{mode!="idle"}[5m])) + +# Top 5 endpoints by request count +topk(5, sum by (endpoint) (rate(http_requests_total[5m]))) + +# 95th percentile latency +histogram_quantile(0.95, rate(http_request_duration_seconds_bucket[5m])) +``` + +### Time-Based Queries + +```promql +# Compare to 1 hour ago +http_requests_total - http_requests_total offset 1h + +# Predict disk space in 4 hours +predict_linear(node_filesystem_avail_bytes[1h], 4 * 3600) + +# Changes in last 5 minutes +changes(up[5m]) + +# Average over 24 hours +avg_over_time(http_requests_total[24h]) +``` + +## Alerting Rules + +```yaml +# rules/alerts.yml +groups: + - name: application + rules: + - alert: HighErrorRate + expr: | + sum(rate(http_requests_total{status=~"5.."}[5m])) + / sum(rate(http_requests_total[5m])) > 0.05 + for: 5m + labels: + severity: critical + annotations: + summary: "High error rate detected" + description: "Error rate is {{ $value | humanizePercentage }}" + + - alert: ServiceDown + expr: up == 0 + for: 1m + labels: + severity: critical + annotations: + summary: "Service {{ $labels.instance }} is down" + + - alert: HighMemoryUsage + expr: | + (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) > 0.9 + for: 5m + labels: + severity: warning + annotations: + summary: "High memory usage on {{ $labels.instance }}" + description: "Memory usage is {{ $value | humanizePercentage }}" + + - alert: DiskSpaceLow + expr: | + (node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) < 0.1 + for: 5m + labels: + severity: warning + annotations: + summary: "Disk space low on {{ $labels.instance }}" +``` + +## Alertmanager + +```yaml +# alertmanager.yml +global: + resolve_timeout: 5m + slack_api_url: 'https://hooks.slack.com/services/xxx' + +route: + receiver: 'slack-notifications' + group_by: ['alertname', 'severity'] + group_wait: 30s + group_interval: 5m + repeat_interval: 4h + routes: + - match: + severity: critical + receiver: 'pagerduty' + +receivers: + - name: 'slack-notifications' + slack_configs: + - channel: '#alerts' + send_resolved: true + title: '{{ .Status | toUpper }}: {{ .CommonAnnotations.summary }}' + text: '{{ .CommonAnnotations.description }}' + + - name: 'pagerduty' + pagerduty_configs: + - service_key: 'xxx' + severity: critical +``` + +## Grafana Dashboards + +### Dashboard JSON Structure + +```json +{ + "dashboard": { + "title": "Application Metrics", + "panels": [ + { + "title": "Request Rate", + "type": "graph", + "targets": [ + { + "expr": "sum(rate(http_requests_total[5m])) by (status_code)", + "legendFormat": "{{ status_code }}" + } + ], + "gridPos": {"x": 0, "y": 0, "w": 12, "h": 8} + }, + { + "title": "Latency P95", + "type": "gauge", + "targets": [ + { + "expr": "histogram_quantile(0.95, rate(http_request_duration_seconds_bucket[5m]))" + } + ], + "gridPos": {"x": 12, "y": 0, "w": 6, "h": 8} + } + ] + } +} +``` + +### Provisioning Dashboards + +```yaml +# grafana/provisioning/dashboards/dashboards.yml +apiVersion: 1 + +providers: + - name: 'default' + orgId: 1 + folder: '' + type: file + disableDeletion: false + updateIntervalSeconds: 30 + options: + path: /var/lib/grafana/dashboards +``` + +### Data Source Provisioning + +```yaml +# grafana/provisioning/datasources/prometheus.yml +apiVersion: 1 + +datasources: + - name: Prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: false +``` + +## Recording Rules + +```yaml +# rules/recording.yml +groups: + - name: aggregations + interval: 30s + rules: + - record: job:http_requests:rate5m + expr: sum by (job) (rate(http_requests_total[5m])) + + - record: instance:node_cpu:avg_rate5m + expr: | + avg by (instance) ( + rate(node_cpu_seconds_total{mode!="idle"}[5m]) + ) + + - record: job:http_latency:p95 + expr: | + histogram_quantile(0.95, + sum by (job, le) (rate(http_request_duration_seconds_bucket[5m])) + ) +``` + +## Application Instrumentation + +### Go Application + +```go +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +var httpRequests = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "http_requests_total", + Help: "Total HTTP requests", + }, + []string{"method", "endpoint", "status"}, +) + +func init() { + prometheus.MustRegister(httpRequests) +} + +// Expose metrics endpoint +http.Handle("/metrics", promhttp.Handler()) +``` + +### Node.js Application + +```javascript +const client = require('prom-client'); + +const httpRequests = new client.Counter({ + name: 'http_requests_total', + help: 'Total HTTP requests', + labelNames: ['method', 'endpoint', 'status'] +}); + +// Middleware +app.use((req, res, next) => { + res.on('finish', () => { + httpRequests.inc({ + method: req.method, + endpoint: req.path, + status: res.statusCode + }); + }); + next(); +}); + +// Expose metrics +app.get('/metrics', async (req, res) => { + res.set('Content-Type', client.register.contentType); + res.end(await client.register.metrics()); +}); +``` + +## Common Issues + +### Issue: Targets Not Discovered +**Problem**: Prometheus not scraping targets +**Solution**: Check network connectivity, verify target labels + +### Issue: High Memory Usage +**Problem**: Prometheus using excessive memory +**Solution**: Reduce retention, use recording rules, limit cardinality + +### Issue: Slow Queries +**Problem**: PromQL queries timing out +**Solution**: Use recording rules, limit time ranges, optimize queries + +### Issue: Missing Data Points +**Problem**: Gaps in metrics data +**Solution**: Check scrape interval, verify target availability + +## Best Practices + +- Use recording rules for frequently-used queries +- Limit label cardinality to prevent memory issues +- Set appropriate retention based on storage capacity +- Use histogram metrics for latency measurement +- Implement proper alerting thresholds +- Version control dashboards as code +- Use federation for large-scale deployments +- Regularly review and prune unused metrics + +## Related Skills + +- [alerting-oncall](../alerting-oncall/) - Alert management +- [loki-logging](../loki-logging/) - Log aggregation +- [kubernetes-ops](../../orchestration/kubernetes-ops/) - K8s monitoring diff --git a/devops/observability/prometheus-grafana/assets/grafana-dashboard-template.json b/devops/observability/prometheus-grafana/assets/grafana-dashboard-template.json new file mode 100644 index 0000000..07287d1 --- /dev/null +++ b/devops/observability/prometheus-grafana/assets/grafana-dashboard-template.json @@ -0,0 +1,198 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "yellow", "value": 70}, + {"color": "red", "value": 90} + ] + }, + "unit": "percent" + } + }, + "gridPos": {"h": 8, "w": 6, "x": 0, "y": 0}, + "id": 1, + "options": { + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "8.0.0", + "targets": [ + { + "expr": "100 - (avg(irate(node_cpu_seconds_total{mode=\"idle\"}[5m])) * 100)", + "refId": "A" + } + ], + "title": "CPU Usage", + "type": "gauge" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": {"mode": "palette-classic"}, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "yellow", "value": 70}, + {"color": "red", "value": 90} + ] + }, + "unit": "percent" + } + }, + "gridPos": {"h": 8, "w": 6, "x": 6, "y": 0}, + "id": 2, + "options": { + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + } + }, + "targets": [ + { + "expr": "(1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100", + "refId": "A" + } + ], + "title": "Memory Usage", + "type": "gauge" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": {"mode": "palette-classic"}, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "lineInterpolation": "linear", + "lineWidth": 1, + "showPoints": "never" + }, + "unit": "reqps" + } + }, + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 0}, + "id": 3, + "options": { + "legend": {"displayMode": "list", "placement": "bottom"}, + "tooltip": {"mode": "single"} + }, + "targets": [ + { + "expr": "sum(rate(http_requests_total[5m])) by (status)", + "legendFormat": "{{status}}", + "refId": "A" + } + ], + "title": "Request Rate by Status", + "type": "timeseries" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": {"mode": "palette-classic"}, + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1 + }, + "unit": "s" + } + }, + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 8}, + "id": 4, + "targets": [ + { + "expr": "histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket[5m])) by (le))", + "legendFormat": "P95", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.50, sum(rate(http_request_duration_seconds_bucket[5m])) by (le))", + "legendFormat": "P50", + "refId": "B" + } + ], + "title": "Request Latency", + "type": "timeseries" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": {"mode": "palette-classic"}, + "custom": { + "drawStyle": "line", + "fillOpacity": 10 + }, + "unit": "percent" + } + }, + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 8}, + "id": 5, + "targets": [ + { + "expr": "sum(rate(http_requests_total{status=~\"5..\"}[5m])) / sum(rate(http_requests_total[5m])) * 100", + "legendFormat": "Error Rate", + "refId": "A" + } + ], + "title": "Error Rate", + "type": "timeseries" + } + ], + "schemaVersion": 30, + "style": "dark", + "tags": ["template", "prometheus"], + "templating": { + "list": [] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Application Dashboard Template", + "version": 1 +} diff --git a/devops/observability/prometheus-grafana/assets/prometheus-config.yaml b/devops/observability/prometheus-grafana/assets/prometheus-config.yaml new file mode 100644 index 0000000..e5d680b --- /dev/null +++ b/devops/observability/prometheus-grafana/assets/prometheus-config.yaml @@ -0,0 +1,115 @@ +# Prometheus Configuration Template +# Customize for your environment + +global: + scrape_interval: 15s + evaluation_interval: 15s + external_labels: + cluster: production + environment: prod + +# Alertmanager configuration +alerting: + alertmanagers: + - static_configs: + - targets: + - alertmanager:9093 + +# Rule files +rule_files: + - /etc/prometheus/rules/*.yaml + +# Scrape configurations +scrape_configs: + + # Prometheus self-monitoring + - job_name: 'prometheus' + static_configs: + - targets: ['localhost:9090'] + + # Node Exporter + - job_name: 'node' + static_configs: + - targets: + - 'node1:9100' + - 'node2:9100' + - 'node3:9100' + + # Kubernetes API Server + - job_name: 'kubernetes-apiservers' + kubernetes_sd_configs: + - role: endpoints + scheme: https + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + relabel_configs: + - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name] + action: keep + regex: default;kubernetes;https + + # Kubernetes Nodes + - job_name: 'kubernetes-nodes' + kubernetes_sd_configs: + - role: node + scheme: https + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + relabel_configs: + - action: labelmap + regex: __meta_kubernetes_node_label_(.+) + + # Kubernetes Pods with prometheus.io annotations + - job_name: 'kubernetes-pods' + kubernetes_sd_configs: + - role: pod + relabel_configs: + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape] + action: keep + regex: true + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path] + action: replace + target_label: __metrics_path__ + regex: (.+) + - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port] + action: replace + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: $1:$2 + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - source_labels: [__meta_kubernetes_namespace] + action: replace + target_label: kubernetes_namespace + - source_labels: [__meta_kubernetes_pod_name] + action: replace + target_label: kubernetes_pod_name + + # Kubernetes Services with prometheus.io annotations + - job_name: 'kubernetes-services' + kubernetes_sd_configs: + - role: service + metrics_path: /probe + params: + module: [http_2xx] + relabel_configs: + - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_probe] + action: keep + regex: true + - source_labels: [__address__] + target_label: __param_target + - target_label: __address__ + replacement: blackbox-exporter:9115 + - source_labels: [__param_target] + target_label: instance + - action: labelmap + regex: __meta_kubernetes_service_label_(.+) + - source_labels: [__meta_kubernetes_namespace] + target_label: kubernetes_namespace + - source_labels: [__meta_kubernetes_service_name] + target_label: kubernetes_name + +# Remote write (optional - for long-term storage) +# remote_write: +# - url: "http://thanos-receive:19291/api/v1/receive" diff --git a/devops/observability/prometheus-grafana/references/alerting-rules.md b/devops/observability/prometheus-grafana/references/alerting-rules.md new file mode 100644 index 0000000..08e7903 --- /dev/null +++ b/devops/observability/prometheus-grafana/references/alerting-rules.md @@ -0,0 +1,170 @@ +# Prometheus Alerting Rules Guide + +## Rule Structure + +```yaml +groups: +- name: example + rules: + - alert: AlertName + expr: + for: + labels: + severity: + team: + annotations: + summary: "Brief description" + description: "Detailed description with {{ $labels.instance }}" + runbook_url: "https://wiki.example.com/alerts/AlertName" +``` + +## Essential Alerts + +### Infrastructure Alerts + +```yaml +groups: +- name: infrastructure + rules: + + # Node down + - alert: NodeDown + expr: up{job="node"} == 0 + for: 5m + labels: + severity: critical + annotations: + summary: "Node {{ $labels.instance }} is down" + + # High CPU + - alert: HighCPU + expr: | + 100 - (avg by (instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80 + for: 10m + labels: + severity: warning + annotations: + summary: "High CPU on {{ $labels.instance }}" + description: "CPU usage is {{ $value | printf \"%.1f\" }}%" + + # High Memory + - alert: HighMemory + expr: | + (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100 > 85 + for: 5m + labels: + severity: warning + annotations: + summary: "High memory on {{ $labels.instance }}" + + # Disk Space Low + - alert: DiskSpaceLow + expr: | + (node_filesystem_avail_bytes{fstype!~"tmpfs|overlay"} + / node_filesystem_size_bytes{fstype!~"tmpfs|overlay"}) * 100 < 15 + for: 5m + labels: + severity: warning + annotations: + summary: "Low disk space on {{ $labels.instance }}" + description: "{{ $labels.mountpoint }} has {{ $value | printf \"%.1f\" }}% free" + + # Disk Space Critical + - alert: DiskSpaceCritical + expr: | + (node_filesystem_avail_bytes{fstype!~"tmpfs|overlay"} + / node_filesystem_size_bytes{fstype!~"tmpfs|overlay"}) * 100 < 5 + for: 1m + labels: + severity: critical + annotations: + summary: "Critical disk space on {{ $labels.instance }}" +``` + +### Application Alerts + +```yaml +groups: +- name: application + rules: + + # High Error Rate + - alert: HighErrorRate + expr: | + sum by (service) (rate(http_requests_total{status=~"5.."}[5m])) + / sum by (service) (rate(http_requests_total[5m])) > 0.05 + for: 5m + labels: + severity: critical + annotations: + summary: "High error rate for {{ $labels.service }}" + description: "Error rate is {{ $value | printf \"%.2f\" }}%" + + # High Latency + - alert: HighLatency + expr: | + histogram_quantile(0.95, + sum by (le, service) (rate(http_request_duration_seconds_bucket[5m])) + ) > 0.5 + for: 5m + labels: + severity: warning + annotations: + summary: "High latency for {{ $labels.service }}" + description: "P95 latency is {{ $value | printf \"%.2f\" }}s" + + # Service Down + - alert: ServiceDown + expr: up{job="app"} == 0 + for: 1m + labels: + severity: critical + annotations: + summary: "Service {{ $labels.instance }} is down" +``` + +### Kubernetes Alerts + +```yaml +groups: +- name: kubernetes + rules: + + # Pod CrashLooping + - alert: PodCrashLooping + expr: | + rate(kube_pod_container_status_restarts_total[15m]) * 60 * 15 > 0 + for: 5m + labels: + severity: warning + annotations: + summary: "Pod {{ $labels.pod }} is crash looping" + + # Pod Not Ready + - alert: PodNotReady + expr: | + kube_pod_status_ready{condition="true"} == 0 + for: 10m + labels: + severity: warning + annotations: + summary: "Pod {{ $labels.pod }} is not ready" + + # Deployment Replicas Mismatch + - alert: DeploymentReplicasMismatch + expr: | + kube_deployment_spec_replicas != kube_deployment_status_replicas_available + for: 10m + labels: + severity: warning + annotations: + summary: "Deployment {{ $labels.deployment }} has replica mismatch" +``` + +## Best Practices + +1. **Use `for` duration** - Avoid alert flapping +2. **Include runbook URLs** - Link to remediation docs +3. **Use severity labels** - Route alerts appropriately +4. **Template annotations** - Include relevant context +5. **Test alerts** - Use `promtool check rules` diff --git a/devops/observability/prometheus-grafana/references/promql-cheatsheet.md b/devops/observability/prometheus-grafana/references/promql-cheatsheet.md new file mode 100644 index 0000000..ae47fdb --- /dev/null +++ b/devops/observability/prometheus-grafana/references/promql-cheatsheet.md @@ -0,0 +1,168 @@ +# PromQL Cheat Sheet + +## Basic Queries + +### Instant Vectors +```promql +# Simple metric +http_requests_total + +# With label filter +http_requests_total{status="200"} + +# Multiple labels +http_requests_total{status="200", method="GET"} + +# Regex matching +http_requests_total{status=~"2.."} +http_requests_total{status!~"5.."} +``` + +### Range Vectors +```promql +# Last 5 minutes +http_requests_total[5m] + +# Last 1 hour +http_requests_total[1h] + +# Time units: s, m, h, d, w, y +``` + +## Functions + +### Rate and Increase +```promql +# Per-second rate over 5m +rate(http_requests_total[5m]) + +# Total increase over 1h +increase(http_requests_total[1h]) + +# For gauges that can decrease +irate(http_requests_total[5m]) # instant rate +``` + +### Aggregations +```promql +# Sum across all instances +sum(http_requests_total) + +# Sum by label +sum by (status) (http_requests_total) + +# Sum excluding label +sum without (instance) (http_requests_total) + +# Other aggregations +avg, min, max, count, stddev, stdvar +topk(5, http_requests_total) +bottomk(3, http_requests_total) +``` + +### Histogram Quantiles +```promql +# 95th percentile +histogram_quantile(0.95, rate(http_request_duration_seconds_bucket[5m])) + +# With grouping +histogram_quantile(0.95, + sum by (le, endpoint) ( + rate(http_request_duration_seconds_bucket[5m]) + ) +) +``` + +## Common Patterns + +### Request Rate +```promql +# Total request rate +sum(rate(http_requests_total[5m])) + +# Request rate by endpoint +sum by (endpoint) (rate(http_requests_total[5m])) +``` + +### Error Rate +```promql +# Error percentage +sum(rate(http_requests_total{status=~"5.."}[5m])) +/ +sum(rate(http_requests_total[5m])) +* 100 +``` + +### Latency +```promql +# Average latency +rate(http_request_duration_seconds_sum[5m]) +/ +rate(http_request_duration_seconds_count[5m]) + +# P99 latency +histogram_quantile(0.99, + sum by (le) (rate(http_request_duration_seconds_bucket[5m])) +) +``` + +### Resource Usage +```promql +# CPU usage percentage +100 - (avg by (instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) + +# Memory usage percentage +(1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100 + +# Disk usage percentage +(1 - (node_filesystem_avail_bytes / node_filesystem_size_bytes)) * 100 +``` + +### Kubernetes +```promql +# Pod CPU usage +sum by (pod) (rate(container_cpu_usage_seconds_total{container!=""}[5m])) + +# Pod memory usage +sum by (pod) (container_memory_usage_bytes{container!=""}) + +# Pod restart count +sum by (pod) (kube_pod_container_status_restarts_total) +``` + +## Alert Examples + +### High Error Rate +```yaml +- alert: HighErrorRate + expr: | + sum(rate(http_requests_total{status=~"5.."}[5m])) + / sum(rate(http_requests_total[5m])) > 0.05 + for: 5m + labels: + severity: critical + annotations: + summary: "High error rate detected" +``` + +### High Latency +```yaml +- alert: HighLatency + expr: | + histogram_quantile(0.95, + sum by (le) (rate(http_request_duration_seconds_bucket[5m])) + ) > 0.5 + for: 5m + labels: + severity: warning +``` + +### Low Disk Space +```yaml +- alert: LowDiskSpace + expr: | + (node_filesystem_avail_bytes / node_filesystem_size_bytes) < 0.1 + for: 5m + labels: + severity: warning +``` diff --git a/devops/observability/prometheus-grafana/scripts/backup-grafana.sh b/devops/observability/prometheus-grafana/scripts/backup-grafana.sh new file mode 100644 index 0000000..2bb4ce6 --- /dev/null +++ b/devops/observability/prometheus-grafana/scripts/backup-grafana.sh @@ -0,0 +1,64 @@ +#!/bin/bash +# Grafana Dashboard Backup Script +# Usage: ./backup-grafana.sh [grafana-url] [api-key] [output-dir] + +set -euo pipefail + +GRAFANA_URL="${1:-http://localhost:3000}" +API_KEY="${2:-$GRAFANA_API_KEY}" +OUTPUT_DIR="${3:-./grafana-backup-$(date +%Y%m%d)}" + +if [ -z "$API_KEY" ]; then + echo "Error: API key required. Set GRAFANA_API_KEY or pass as argument." + exit 1 +fi + +mkdir -p "$OUTPUT_DIR/dashboards" +mkdir -p "$OUTPUT_DIR/datasources" +mkdir -p "$OUTPUT_DIR/folders" + +echo "=========================================" +echo "Grafana Backup" +echo "URL: $GRAFANA_URL" +echo "Output: $OUTPUT_DIR" +echo "=========================================" +echo "" + +# Backup datasources +echo "Backing up datasources..." +curl -s -H "Authorization: Bearer $API_KEY" \ + "$GRAFANA_URL/api/datasources" > "$OUTPUT_DIR/datasources/datasources.json" +DS_COUNT=$(jq length "$OUTPUT_DIR/datasources/datasources.json") +echo " Backed up $DS_COUNT datasources" + +# Backup folders +echo "Backing up folders..." +curl -s -H "Authorization: Bearer $API_KEY" \ + "$GRAFANA_URL/api/folders" > "$OUTPUT_DIR/folders/folders.json" +FOLDER_COUNT=$(jq length "$OUTPUT_DIR/folders/folders.json") +echo " Backed up $FOLDER_COUNT folders" + +# Get all dashboards +echo "Backing up dashboards..." +DASHBOARDS=$(curl -s -H "Authorization: Bearer $API_KEY" \ + "$GRAFANA_URL/api/search?type=dash-db") + +DASH_COUNT=0 +echo "$DASHBOARDS" | jq -r '.[].uid' | while read uid; do + DASH=$(curl -s -H "Authorization: Bearer $API_KEY" \ + "$GRAFANA_URL/api/dashboards/uid/$uid") + + TITLE=$(echo "$DASH" | jq -r '.dashboard.title' | tr ' /' '_') + echo "$DASH" > "$OUTPUT_DIR/dashboards/${TITLE}_${uid}.json" + echo " - $TITLE" + DASH_COUNT=$((DASH_COUNT + 1)) +done + +echo "" +echo "Backup complete!" +echo "Location: $OUTPUT_DIR" +echo "" +echo "To restore:" +echo " 1. Datasources: POST to /api/datasources" +echo " 2. Folders: POST to /api/folders" +echo " 3. Dashboards: POST to /api/dashboards/db" diff --git a/devops/observability/prometheus-grafana/scripts/prometheus-health-check.sh b/devops/observability/prometheus-grafana/scripts/prometheus-health-check.sh new file mode 100644 index 0000000..8728582 --- /dev/null +++ b/devops/observability/prometheus-grafana/scripts/prometheus-health-check.sh @@ -0,0 +1,82 @@ +#!/bin/bash +# Prometheus Health Check Script +# Usage: ./prometheus-health-check.sh [prometheus-url] + +set -euo pipefail + +PROMETHEUS_URL="${1:-http://localhost:9090}" + +echo "=========================================" +echo "Prometheus Health Check" +echo "URL: $PROMETHEUS_URL" +echo "=========================================" +echo "" + +# Check Prometheus health +echo -n "Prometheus Health: " +HEALTH=$(curl -s "$PROMETHEUS_URL/-/healthy" 2>/dev/null) +if [ "$HEALTH" == "Prometheus Server is Healthy." ]; then + echo "✓ Healthy" +else + echo "✗ Unhealthy" +fi + +# Check readiness +echo -n "Prometheus Ready: " +READY=$(curl -s "$PROMETHEUS_URL/-/ready" 2>/dev/null) +if [ "$READY" == "Prometheus Server is Ready." ]; then + echo "✓ Ready" +else + echo "✗ Not Ready" +fi + +# Get runtime info +echo "" +echo "Runtime Information:" +echo "--------------------" +curl -s "$PROMETHEUS_URL/api/v1/status/runtimeinfo" 2>/dev/null | jq -r ' + .data | + "Start Time: \(.startTime)", + "Uptime: \(.CWD // "N/A")", + "Storage Retention: \(.storageRetention)", + "TSDB Info:", + " - Head Chunks: \(.TSDB.headChunks // "N/A")", + " - Head Series: \(.TSDB.headSeries // "N/A")" +' 2>/dev/null || echo "Could not retrieve runtime info" + +# Get active targets summary +echo "" +echo "Target Status:" +echo "--------------" +curl -s "$PROMETHEUS_URL/api/v1/targets" 2>/dev/null | jq -r ' + .data.activeTargets | + group_by(.health) | + map({health: .[0].health, count: length}) | + .[] | + "\(.health): \(.count) targets" +' 2>/dev/null || echo "Could not retrieve targets" + +# List unhealthy targets +echo "" +echo "Unhealthy Targets:" +echo "------------------" +curl -s "$PROMETHEUS_URL/api/v1/targets" 2>/dev/null | jq -r ' + .data.activeTargets[] | + select(.health != "up") | + "- \(.labels.job)/\(.labels.instance): \(.lastError)" +' 2>/dev/null || echo "Could not check unhealthy targets" + +# Check for firing alerts +echo "" +echo "Firing Alerts:" +echo "--------------" +curl -s "$PROMETHEUS_URL/api/v1/alerts" 2>/dev/null | jq -r ' + .data.alerts[] | + select(.state == "firing") | + "- [\(.labels.severity // "unknown")] \(.labels.alertname): \(.annotations.summary // .annotations.description // "No description")" +' 2>/dev/null || echo "Could not retrieve alerts" + +echo "" +echo "=========================================" +echo "Health check complete" +echo "=========================================" diff --git a/devops/orchestration/argocd-gitops/SKILL.md b/devops/orchestration/argocd-gitops/SKILL.md new file mode 100644 index 0000000..6351b04 --- /dev/null +++ b/devops/orchestration/argocd-gitops/SKILL.md @@ -0,0 +1,445 @@ +--- +name: argocd-gitops +description: Implement GitOps with ArgoCD for declarative Kubernetes deployments. Configure applications, manage sync policies, implement progressive delivery, and automate deployments from Git repositories. Use when implementing GitOps workflows or continuous deployment to Kubernetes. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# ArgoCD GitOps + +Implement declarative continuous delivery for Kubernetes with ArgoCD. + +## When to Use This Skill + +Use this skill when: +- Implementing GitOps workflows for Kubernetes +- Automating deployments from Git repositories +- Managing multiple environments declaratively +- Implementing progressive delivery strategies +- Synchronizing cluster state with Git + +## Prerequisites + +- Kubernetes cluster with ArgoCD installed +- kubectl configured +- Git repository for manifests +- ArgoCD CLI (optional) + +## Installation + +```bash +# Create namespace +kubectl create namespace argocd + +# Install ArgoCD +kubectl apply -n argocd -f https://raw.githubusercontent.com/argoproj/argo-cd/stable/manifests/install.yaml + +# Get admin password +kubectl -n argocd get secret argocd-initial-admin-secret \ + -o jsonpath="{.data.password}" | base64 -d + +# Port forward to access UI +kubectl port-forward svc/argocd-server -n argocd 8080:443 + +# Login with CLI +argocd login localhost:8080 +``` + +## Application Definition + +### Basic Application + +```yaml +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: myapp + namespace: argocd +spec: + project: default + source: + repoURL: https://github.com/org/myapp-manifests.git + targetRevision: main + path: environments/production + destination: + server: https://kubernetes.default.svc + namespace: myapp + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true +``` + +### Helm Application + +```yaml +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: myapp-helm + namespace: argocd +spec: + project: default + source: + repoURL: https://github.com/org/myapp-chart.git + targetRevision: main + path: charts/myapp + helm: + valueFiles: + - values.yaml + - values-production.yaml + parameters: + - name: replicaCount + value: "3" + - name: image.tag + value: "2.0.0" + destination: + server: https://kubernetes.default.svc + namespace: myapp + syncPolicy: + automated: + prune: true + selfHeal: true +``` + +### Kustomize Application + +```yaml +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: myapp-kustomize + namespace: argocd +spec: + project: default + source: + repoURL: https://github.com/org/myapp-manifests.git + targetRevision: main + path: overlays/production + kustomize: + images: + - myapp=myregistry/myapp:2.0.0 + destination: + server: https://kubernetes.default.svc + namespace: myapp +``` + +## Projects + +```yaml +apiVersion: argoproj.io/v1alpha1 +kind: AppProject +metadata: + name: myproject + namespace: argocd +spec: + description: My Project + sourceRepos: + - https://github.com/org/* + destinations: + - namespace: myapp-* + server: https://kubernetes.default.svc + clusterResourceWhitelist: + - group: '' + kind: Namespace + namespaceResourceWhitelist: + - group: '*' + kind: '*' + roles: + - name: developer + description: Developer role + policies: + - p, proj:myproject:developer, applications, get, myproject/*, allow + - p, proj:myproject:developer, applications, sync, myproject/*, allow + groups: + - developers +``` + +## Application Sets + +### Git Generator + +```yaml +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: myapp-environments + namespace: argocd +spec: + generators: + - git: + repoURL: https://github.com/org/myapp-manifests.git + revision: main + directories: + - path: environments/* + template: + metadata: + name: 'myapp-{{path.basename}}' + spec: + project: default + source: + repoURL: https://github.com/org/myapp-manifests.git + targetRevision: main + path: '{{path}}' + destination: + server: https://kubernetes.default.svc + namespace: 'myapp-{{path.basename}}' + syncPolicy: + automated: + prune: true + selfHeal: true +``` + +### List Generator + +```yaml +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: myapp-clusters + namespace: argocd +spec: + generators: + - list: + elements: + - cluster: production + url: https://prod-cluster.example.com + - cluster: staging + url: https://staging-cluster.example.com + template: + metadata: + name: 'myapp-{{cluster}}' + spec: + project: default + source: + repoURL: https://github.com/org/myapp-manifests.git + targetRevision: main + path: 'environments/{{cluster}}' + destination: + server: '{{url}}' + namespace: myapp +``` + +### Matrix Generator + +```yaml +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: myapp-matrix + namespace: argocd +spec: + generators: + - matrix: + generators: + - git: + repoURL: https://github.com/org/myapp-manifests.git + revision: main + directories: + - path: apps/* + - list: + elements: + - env: staging + - env: production + template: + metadata: + name: '{{path.basename}}-{{env}}' + spec: + project: default + source: + repoURL: https://github.com/org/myapp-manifests.git + targetRevision: main + path: '{{path}}/overlays/{{env}}' + destination: + server: https://kubernetes.default.svc + namespace: '{{path.basename}}-{{env}}' +``` + +## Sync Policies + +### Automated Sync + +```yaml +syncPolicy: + automated: + prune: true # Delete resources not in Git + selfHeal: true # Revert manual changes + allowEmpty: false # Don't sync empty directories + syncOptions: + - CreateNamespace=true + - PrunePropagationPolicy=foreground + - PruneLast=true + retry: + limit: 5 + backoff: + duration: 5s + factor: 2 + maxDuration: 3m +``` + +### Sync Waves + +```yaml +# In Kubernetes manifests +apiVersion: v1 +kind: ConfigMap +metadata: + name: myconfig + annotations: + argocd.argoproj.io/sync-wave: "-1" # Sync first +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: myapp + annotations: + argocd.argoproj.io/sync-wave: "0" # Sync second +``` + +### Sync Hooks + +```yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: migration + annotations: + argocd.argoproj.io/hook: PreSync + argocd.argoproj.io/hook-delete-policy: HookSucceeded +spec: + template: + spec: + containers: + - name: migrate + image: myapp:latest + command: ["./migrate.sh"] + restartPolicy: Never +``` + +## CLI Commands + +```bash +# List applications +argocd app list + +# Get application details +argocd app get myapp + +# Sync application +argocd app sync myapp + +# Force sync (ignore differences) +argocd app sync myapp --force + +# View diff +argocd app diff myapp + +# Rollback +argocd app rollback myapp + +# Delete application +argocd app delete myapp + +# View logs +argocd app logs myapp + +# Hard refresh (clear cache) +argocd app get myapp --hard-refresh +``` + +## Repository Configuration + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: private-repo + namespace: argocd + labels: + argocd.argoproj.io/secret-type: repository +stringData: + url: https://github.com/org/private-repo.git + username: git + password: ghp_xxxx +--- +# SSH key +apiVersion: v1 +kind: Secret +metadata: + name: private-repo-ssh + namespace: argocd + labels: + argocd.argoproj.io/secret-type: repository +stringData: + url: git@github.com:org/private-repo.git + sshPrivateKey: | + -----BEGIN OPENSSH PRIVATE KEY----- + ... + -----END OPENSSH PRIVATE KEY----- +``` + +## Notifications + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: argocd-notifications-cm + namespace: argocd +data: + service.slack: | + token: $slack-token + template.app-deployed: | + message: Application {{.app.metadata.name}} is now {{.app.status.sync.status}}. + trigger.on-deployed: | + - when: app.status.operationState.phase in ['Succeeded'] + send: [app-deployed] +``` + +## Common Issues + +### Issue: Sync Fails with Diff +**Problem**: Resources show differences but are correct +**Solution**: Configure ignore differences + +```yaml +spec: + ignoreDifferences: + - group: apps + kind: Deployment + jsonPointers: + - /spec/replicas +``` + +### Issue: Repository Not Accessible +**Problem**: ArgoCD cannot clone repository +**Solution**: Check repository secret, verify URL and credentials + +### Issue: Application Stuck OutOfSync +**Problem**: Application never becomes synced +**Solution**: Check resource status, review events, verify manifests + +### Issue: Health Check Failing +**Problem**: Application shows degraded health +**Solution**: Check custom health checks, verify probe configurations + +## Best Practices + +- Use ApplicationSets for multi-environment deployments +- Implement sync waves for ordered deployments +- Use projects to isolate applications +- Configure notifications for deployment events +- Implement proper RBAC with projects +- Use health checks for deployment verification +- Enable auto-pruning to remove deleted resources +- Keep manifests in dedicated repositories + +## Related Skills + +- [kubernetes-ops](../kubernetes-ops/) - K8s fundamentals +- [helm-charts](../helm-charts/) - Helm deployments +- [kustomize](../kustomize/) - Kustomize overlays diff --git a/devops/orchestration/argocd-gitops/references/argocd-patterns.md b/devops/orchestration/argocd-gitops/references/argocd-patterns.md new file mode 100644 index 0000000..3135d0c --- /dev/null +++ b/devops/orchestration/argocd-gitops/references/argocd-patterns.md @@ -0,0 +1,126 @@ +# ArgoCD GitOps Patterns + +## Application Definition + +```yaml +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: myapp + namespace: argocd +spec: + project: default + source: + repoURL: https://github.com/org/repo + targetRevision: HEAD + path: k8s/overlays/production + destination: + server: https://kubernetes.default.svc + namespace: myapp + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true +``` + +## Sync Waves + +```yaml +metadata: + annotations: + argocd.argoproj.io/sync-wave: "-1" # Deploy first +--- +metadata: + annotations: + argocd.argoproj.io/sync-wave: "0" # Default +--- +metadata: + annotations: + argocd.argoproj.io/sync-wave: "1" # Deploy last +``` + +## ApplicationSet + +```yaml +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: myapp-set +spec: + generators: + - list: + elements: + - env: dev + cluster: dev-cluster + - env: prod + cluster: prod-cluster + template: + metadata: + name: 'myapp-{{env}}' + spec: + source: + repoURL: https://github.com/org/repo + path: 'k8s/overlays/{{env}}' + destination: + server: '{{cluster}}' + namespace: myapp +``` + +## Multi-Cluster + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: prod-cluster + labels: + argocd.argoproj.io/secret-type: cluster +stringData: + name: prod + server: https://prod-cluster.example.com + config: | + { + "bearerToken": "...", + "tlsClientConfig": { + "insecure": false, + "caData": "..." + } + } +``` + +## App of Apps + +```yaml +# Root application +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: root +spec: + source: + path: apps/ # Contains Application manifests + destination: + namespace: argocd +``` + +## CLI Commands + +```bash +# Login +argocd login argocd.example.com + +# Sync +argocd app sync myapp +argocd app sync myapp --prune + +# Rollback +argocd app rollback myapp + +# Diff +argocd app diff myapp + +# History +argocd app history myapp +``` diff --git a/devops/orchestration/helm-charts/SKILL.md b/devops/orchestration/helm-charts/SKILL.md new file mode 100644 index 0000000..153baff --- /dev/null +++ b/devops/orchestration/helm-charts/SKILL.md @@ -0,0 +1,445 @@ +--- +name: helm-charts +description: Create, manage, and deploy Helm charts for Kubernetes package management. Build reusable chart templates, manage releases, configure values, and use Helm repositories. Use when packaging Kubernetes applications or managing K8s deployments with Helm. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# Helm Charts + +Package and deploy Kubernetes applications using Helm, the package manager for Kubernetes. + +## When to Use This Skill + +Use this skill when: +- Creating reusable Kubernetes application packages +- Deploying applications with configurable values +- Managing Helm releases and upgrades +- Using third-party Helm charts +- Implementing chart versioning and repositories + +## Prerequisites + +- Helm 3.x installed +- kubectl configured with cluster access +- Basic Kubernetes knowledge + +## Chart Structure + +``` +mychart/ +├── Chart.yaml # Chart metadata +├── values.yaml # Default configuration values +├── charts/ # Chart dependencies +├── templates/ # Kubernetes manifest templates +│ ├── deployment.yaml +│ ├── service.yaml +│ ├── ingress.yaml +│ ├── configmap.yaml +│ ├── secret.yaml +│ ├── _helpers.tpl # Template helpers +│ ├── NOTES.txt # Post-install notes +│ └── tests/ +│ └── test-connection.yaml +└── .helmignore # Files to ignore +``` + +## Chart.yaml + +```yaml +apiVersion: v2 +name: myapp +description: A Helm chart for MyApp +type: application +version: 1.0.0 +appVersion: "2.0.0" +keywords: + - myapp + - web +maintainers: + - name: DevOps Team + email: devops@example.com +dependencies: + - name: postgresql + version: "12.x.x" + repository: "https://charts.bitnami.com/bitnami" + condition: postgresql.enabled +``` + +## values.yaml + +```yaml +replicaCount: 2 + +image: + repository: myapp + tag: "" # Defaults to appVersion + pullPolicy: IfNotPresent + +service: + type: ClusterIP + port: 80 + +ingress: + enabled: false + className: nginx + hosts: + - host: myapp.example.com + paths: + - path: / + pathType: Prefix + tls: [] + +resources: + limits: + cpu: 500m + memory: 256Mi + requests: + cpu: 100m + memory: 128Mi + +postgresql: + enabled: true + auth: + database: myapp +``` + +## Templates + +### Deployment Template + +```yaml +# templates/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "myapp.fullname" . }} + labels: + {{- include "myapp.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + {{- include "myapp.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "myapp.selectorLabels" . | nindent 8 }} + spec: + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - name: http + containerPort: 8080 + {{- with .Values.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + env: + - name: DATABASE_URL + valueFrom: + secretKeyRef: + name: {{ include "myapp.fullname" . }}-secrets + key: database-url +``` + +### Helper Functions + +```yaml +# templates/_helpers.tpl +{{/* +Expand the name of the chart. +*/}} +{{- define "myapp.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +*/}} +{{- define "myapp.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "myapp.labels" -}} +helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }} +{{ include "myapp.selectorLabels" . }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "myapp.selectorLabels" -}} +app.kubernetes.io/name: {{ include "myapp.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} +``` + +### Conditional Resources + +```yaml +# templates/ingress.yaml +{{- if .Values.ingress.enabled -}} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ include "myapp.fullname" . }} + labels: + {{- include "myapp.labels" . | nindent 4 }} + {{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + ingressClassName: {{ .Values.ingress.className }} + {{- if .Values.ingress.tls }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + pathType: {{ .pathType }} + backend: + service: + name: {{ include "myapp.fullname" $ }} + port: + number: {{ $.Values.service.port }} + {{- end }} + {{- end }} +{{- end }} +``` + +## Helm Commands + +### Installing Charts + +```bash +# Install from local chart +helm install myapp ./mychart + +# Install with custom values +helm install myapp ./mychart -f custom-values.yaml + +# Install with value overrides +helm install myapp ./mychart \ + --set replicaCount=3 \ + --set image.tag=2.0.0 + +# Install in specific namespace +helm install myapp ./mychart -n production --create-namespace + +# Dry run to preview +helm install myapp ./mychart --dry-run --debug +``` + +### Managing Releases + +```bash +# List releases +helm list +helm list -A # All namespaces + +# Upgrade release +helm upgrade myapp ./mychart +helm upgrade myapp ./mychart -f new-values.yaml + +# Rollback +helm rollback myapp 1 +helm history myapp + +# Uninstall +helm uninstall myapp +``` + +### Chart Development + +```bash +# Create new chart +helm create mychart + +# Lint chart +helm lint ./mychart + +# Template locally (debug) +helm template myapp ./mychart + +# Package chart +helm package ./mychart + +# Update dependencies +helm dependency update ./mychart +``` + +## Repositories + +```bash +# Add repository +helm repo add bitnami https://charts.bitnami.com/bitnami + +# Update repositories +helm repo update + +# Search charts +helm search repo postgresql +helm search hub prometheus + +# Install from repo +helm install postgres bitnami/postgresql + +# Show chart info +helm show values bitnami/postgresql +``` + +## Advanced Features + +### Hooks + +```yaml +# templates/pre-install-job.yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "myapp.fullname" . }}-migration + annotations: + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-weight": "-5" + "helm.sh/hook-delete-policy": hook-succeeded +spec: + template: + spec: + containers: + - name: migrate + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + command: ["./migrate.sh"] + restartPolicy: Never +``` + +### Tests + +```yaml +# templates/tests/test-connection.yaml +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "myapp.fullname" . }}-test-connection" + annotations: + "helm.sh/hook": test +spec: + containers: + - name: wget + image: busybox + command: ['wget'] + args: ['{{ include "myapp.fullname" . }}:{{ .Values.service.port }}'] + restartPolicy: Never +``` + +```bash +# Run tests +helm test myapp +``` + +### Library Charts + +```yaml +# Chart.yaml +apiVersion: v2 +name: mylib +type: library +version: 1.0.0 +``` + +```yaml +# Using library chart +dependencies: + - name: mylib + version: "1.x.x" + repository: "file://../mylib" +``` + +## OCI Registry Support + +```bash +# Login to registry +helm registry login registry.example.com + +# Push chart to OCI registry +helm push mychart-1.0.0.tgz oci://registry.example.com/charts + +# Pull from OCI registry +helm pull oci://registry.example.com/charts/mychart --version 1.0.0 + +# Install from OCI +helm install myapp oci://registry.example.com/charts/mychart +``` + +## Common Issues + +### Issue: YAML Indentation Errors +**Problem**: Template renders with wrong indentation +**Solution**: Use `nindent` helper function + +```yaml +{{- toYaml .Values.resources | nindent 12 }} +``` + +### Issue: Values Not Applying +**Problem**: Custom values not reflected +**Solution**: Check value paths, use `--debug` flag + +```bash +helm template myapp ./mychart --debug +``` + +### Issue: Dependency Errors +**Problem**: Chart dependencies not found +**Solution**: Run `helm dependency update` + +### Issue: Release Already Exists +**Problem**: Cannot install, release exists +**Solution**: Use `helm upgrade --install` + +```bash +helm upgrade --install myapp ./mychart +``` + +## Best Practices + +- Use semantic versioning for charts +- Provide comprehensive default values +- Document all values in values.yaml with comments +- Use helper templates for repeated patterns +- Implement chart tests +- Use .helmignore to exclude unnecessary files +- Pin dependency versions +- Use `helm lint` in CI pipelines + +## Related Skills + +- [kubernetes-ops](../kubernetes-ops/) - K8s fundamentals +- [argocd-gitops](../argocd-gitops/) - GitOps with Helm +- [kustomize](../kustomize/) - Alternative templating diff --git a/devops/orchestration/helm-charts/references/helm-commands.md b/devops/orchestration/helm-charts/references/helm-commands.md new file mode 100644 index 0000000..4709c7f --- /dev/null +++ b/devops/orchestration/helm-charts/references/helm-commands.md @@ -0,0 +1,121 @@ +# Helm Commands Reference + +## Chart Management + +```bash +# Create new chart +helm create mychart + +# Lint chart +helm lint mychart/ + +# Package chart +helm package mychart/ + +# Template (render without installing) +helm template myrelease mychart/ --values values.yaml +``` + +## Repository + +```bash +# Add repo +helm repo add bitnami https://charts.bitnami.com/bitnami +helm repo add stable https://charts.helm.sh/stable + +# Update repos +helm repo update + +# Search +helm search repo nginx +helm search hub nginx +``` + +## Installation + +```bash +# Install +helm install myrelease mychart/ +helm install myrelease bitnami/nginx --namespace prod --create-namespace + +# With values +helm install myrelease mychart/ -f values.yaml +helm install myrelease mychart/ --set image.tag=v1.0 + +# Dry run +helm install myrelease mychart/ --dry-run --debug + +# Wait for completion +helm install myrelease mychart/ --wait --timeout 5m +``` + +## Upgrade & Rollback + +```bash +# Upgrade +helm upgrade myrelease mychart/ -f values.yaml +helm upgrade --install myrelease mychart/ # Install or upgrade + +# Rollback +helm rollback myrelease 1 # Rollback to revision 1 +helm rollback myrelease # Previous revision + +# History +helm history myrelease +``` + +## Management + +```bash +# List releases +helm list +helm list -A # All namespaces +helm list --pending + +# Get info +helm get values myrelease +helm get manifest myrelease +helm get all myrelease + +# Status +helm status myrelease + +# Uninstall +helm uninstall myrelease +helm uninstall myrelease --keep-history +``` + +## Chart Structure + +``` +mychart/ +├── Chart.yaml # Chart metadata +├── values.yaml # Default values +├── charts/ # Dependencies +├── templates/ +│ ├── NOTES.txt # Post-install notes +│ ├── _helpers.tpl # Template helpers +│ ├── deployment.yaml +│ ├── service.yaml +│ └── ingress.yaml +└── .helmignore +``` + +## Template Functions + +```yaml +# Built-in functions +{{ .Values.image.tag | default "latest" }} +{{ .Release.Name | upper }} +{{ include "mychart.fullname" . }} + +# Conditionals +{{- if .Values.ingress.enabled }} +# ingress config +{{- end }} + +# Loops +{{- range .Values.hosts }} +- host: {{ . }} +{{- end }} +``` diff --git a/devops/orchestration/kubernetes-ops/SKILL.md b/devops/orchestration/kubernetes-ops/SKILL.md new file mode 100644 index 0000000..1607077 --- /dev/null +++ b/devops/orchestration/kubernetes-ops/SKILL.md @@ -0,0 +1,516 @@ +--- +name: kubernetes-ops +description: Deploy, scale, and manage Kubernetes workloads. Create deployments, services, and configurations, manage cluster resources, troubleshoot pods, and implement production-ready Kubernetes patterns. Use when working with Kubernetes clusters, K8s deployments, or container orchestration. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# Kubernetes Operations + +Deploy and manage containerized applications on Kubernetes clusters. + +## When to Use This Skill + +Use this skill when: +- Deploying applications to Kubernetes +- Managing pods, deployments, and services +- Configuring resource limits and scaling +- Troubleshooting Kubernetes workloads +- Setting up networking and ingress + +## Prerequisites + +- kubectl installed and configured +- Access to a Kubernetes cluster +- Basic understanding of containers + +## Core Resources + +### Deployment + +```yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: myapp + labels: + app: myapp +spec: + replicas: 3 + selector: + matchLabels: + app: myapp + template: + metadata: + labels: + app: myapp + spec: + containers: + - name: myapp + image: myapp:1.0.0 + ports: + - containerPort: 8080 + resources: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "256Mi" + cpu: "500m" + livenessProbe: + httpGet: + path: /health + port: 8080 + initialDelaySeconds: 10 + periodSeconds: 10 + readinessProbe: + httpGet: + path: /ready + port: 8080 + initialDelaySeconds: 5 + periodSeconds: 5 + env: + - name: DATABASE_URL + valueFrom: + secretKeyRef: + name: myapp-secrets + key: database-url +``` + +### Service + +```yaml +apiVersion: v1 +kind: Service +metadata: + name: myapp +spec: + selector: + app: myapp + ports: + - port: 80 + targetPort: 8080 + type: ClusterIP +--- +# LoadBalancer for external access +apiVersion: v1 +kind: Service +metadata: + name: myapp-external +spec: + selector: + app: myapp + ports: + - port: 80 + targetPort: 8080 + type: LoadBalancer +``` + +### Ingress + +```yaml +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: myapp + annotations: + nginx.ingress.kubernetes.io/rewrite-target: / +spec: + ingressClassName: nginx + tls: + - hosts: + - myapp.example.com + secretName: myapp-tls + rules: + - host: myapp.example.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: myapp + port: + number: 80 +``` + +## Configuration Management + +### ConfigMap + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: myapp-config +data: + config.yaml: | + server: + port: 8080 + logging: + level: info + APP_ENV: production +``` + +```yaml +# Using ConfigMap +containers: +- name: myapp + envFrom: + - configMapRef: + name: myapp-config + volumeMounts: + - name: config + mountPath: /etc/config +volumes: +- name: config + configMap: + name: myapp-config +``` + +### Secret + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: myapp-secrets +type: Opaque +stringData: + database-url: postgres://user:pass@host:5432/db + api-key: secret-key-value +``` + +```bash +# Create secret from command line +kubectl create secret generic myapp-secrets \ + --from-literal=database-url='postgres://...' \ + --from-file=tls.crt=cert.pem +``` + +## kubectl Commands + +### Resource Management + +```bash +# Apply configuration +kubectl apply -f deployment.yaml + +# Get resources +kubectl get pods +kubectl get deployments +kubectl get services +kubectl get all -n myapp + +# Describe resource +kubectl describe pod myapp-xxx + +# Delete resource +kubectl delete -f deployment.yaml +kubectl delete pod myapp-xxx + +# Edit resource +kubectl edit deployment myapp +``` + +### Debugging + +```bash +# View logs +kubectl logs myapp-xxx +kubectl logs -f myapp-xxx --tail=100 +kubectl logs myapp-xxx -c sidecar # specific container + +# Execute command +kubectl exec -it myapp-xxx -- /bin/sh + +# Port forward +kubectl port-forward svc/myapp 8080:80 +kubectl port-forward pod/myapp-xxx 8080:8080 + +# View events +kubectl get events --sort-by='.lastTimestamp' + +# Debug pod +kubectl debug myapp-xxx -it --image=busybox +``` + +### Scaling + +```bash +# Manual scaling +kubectl scale deployment myapp --replicas=5 + +# Autoscaling +kubectl autoscale deployment myapp \ + --min=2 --max=10 \ + --cpu-percent=80 +``` + +## Horizontal Pod Autoscaler + +```yaml +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: myapp +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: myapp + minReplicas: 2 + maxReplicas: 10 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 80 + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: 80 +``` + +## Persistent Storage + +### PersistentVolumeClaim + +```yaml +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: myapp-data +spec: + accessModes: + - ReadWriteOnce + storageClassName: standard + resources: + requests: + storage: 10Gi +--- +# Using PVC +containers: +- name: myapp + volumeMounts: + - name: data + mountPath: /data +volumes: +- name: data + persistentVolumeClaim: + claimName: myapp-data +``` + +## StatefulSet + +```yaml +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: postgres +spec: + serviceName: postgres + replicas: 3 + selector: + matchLabels: + app: postgres + template: + metadata: + labels: + app: postgres + spec: + containers: + - name: postgres + image: postgres:15 + ports: + - containerPort: 5432 + volumeMounts: + - name: data + mountPath: /var/lib/postgresql/data + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 10Gi +``` + +## Jobs and CronJobs + +### Job + +```yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: migration +spec: + template: + spec: + containers: + - name: migrate + image: myapp:1.0.0 + command: ["./migrate.sh"] + restartPolicy: Never + backoffLimit: 3 +``` + +### CronJob + +```yaml +apiVersion: batch/v1 +kind: CronJob +metadata: + name: backup +spec: + schedule: "0 2 * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: backup + image: backup-tool:latest + command: ["./backup.sh"] + restartPolicy: OnFailure +``` + +## Network Policies + +```yaml +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: myapp-network-policy +spec: + podSelector: + matchLabels: + app: myapp + policyTypes: + - Ingress + - Egress + ingress: + - from: + - podSelector: + matchLabels: + app: frontend + ports: + - protocol: TCP + port: 8080 + egress: + - to: + - podSelector: + matchLabels: + app: database + ports: + - protocol: TCP + port: 5432 +``` + +## Resource Quotas + +```yaml +apiVersion: v1 +kind: ResourceQuota +metadata: + name: myapp-quota + namespace: myapp +spec: + hard: + requests.cpu: "10" + requests.memory: 20Gi + limits.cpu: "20" + limits.memory: 40Gi + pods: "20" +``` + +## Rolling Updates + +```yaml +spec: + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 +``` + +```bash +# Update image +kubectl set image deployment/myapp myapp=myapp:2.0.0 + +# Check rollout status +kubectl rollout status deployment/myapp + +# View history +kubectl rollout history deployment/myapp + +# Rollback +kubectl rollout undo deployment/myapp +kubectl rollout undo deployment/myapp --to-revision=2 +``` + +## Common Issues + +### Issue: Pod Stuck in Pending +**Problem**: Pod won't start +**Solution**: Check resource availability, node selector, PVC binding + +```bash +kubectl describe pod myapp-xxx +kubectl get events +``` + +### Issue: CrashLoopBackOff +**Problem**: Container keeps restarting +**Solution**: Check logs, verify entrypoint, check probes + +```bash +kubectl logs myapp-xxx --previous +kubectl describe pod myapp-xxx +``` + +### Issue: Service Not Accessible +**Problem**: Cannot connect to service +**Solution**: Check selector labels, verify endpoints exist + +```bash +kubectl get endpoints myapp +kubectl describe svc myapp +``` + +### Issue: Image Pull Error +**Problem**: ImagePullBackOff +**Solution**: Check image name, verify registry credentials + +```bash +kubectl create secret docker-registry regcred \ + --docker-server=registry.example.com \ + --docker-username=user \ + --docker-password=pass +``` + +## Best Practices + +- Always set resource requests and limits +- Implement liveness and readiness probes +- Use namespaces for isolation +- Apply network policies for security +- Use ConfigMaps and Secrets for configuration +- Implement pod disruption budgets for availability +- Use labels consistently for organization +- Enable RBAC for access control + +## Related Skills + +- [helm-charts](../helm-charts/) - Package management +- [argocd-gitops](../argocd-gitops/) - GitOps deployments +- [kubernetes-hardening](../../../security/hardening/kubernetes-hardening/) - Security diff --git a/devops/orchestration/kubernetes-ops/assets/deployment-template.yaml b/devops/orchestration/kubernetes-ops/assets/deployment-template.yaml new file mode 100644 index 0000000..0dac833 --- /dev/null +++ b/devops/orchestration/kubernetes-ops/assets/deployment-template.yaml @@ -0,0 +1,166 @@ +# Production-Ready Deployment Template +# Customize values marked with + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: + labels: + app: + version: v1 +spec: + replicas: 3 + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + selector: + matchLabels: + app: + template: + metadata: + labels: + app: + version: v1 + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "8080" + prometheus.io/path: "/metrics" + spec: + serviceAccountName: + + securityContext: + runAsNonRoot: true + runAsUser: 1000 + fsGroup: 1000 + + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchLabels: + app: + topologyKey: kubernetes.io/hostname + + containers: + - name: + image: : + imagePullPolicy: Always + + ports: + - name: http + containerPort: 8080 + protocol: TCP + + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + + envFrom: + - configMapRef: + name: -config + - secretRef: + name: -secrets + + resources: + requests: + memory: "256Mi" + cpu: "250m" + limits: + memory: "512Mi" + + securityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + + livenessProbe: + httpGet: + path: /healthz + port: http + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + + readinessProbe: + httpGet: + path: /ready + port: http + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 3 + + volumeMounts: + - name: tmp + mountPath: /tmp + - name: cache + mountPath: /var/cache + + volumes: + - name: tmp + emptyDir: {} + - name: cache + emptyDir: {} + + terminationGracePeriodSeconds: 30 + +--- +apiVersion: v1 +kind: Service +metadata: + name: + labels: + app: +spec: + type: ClusterIP + ports: + - port: 80 + targetPort: http + protocol: TCP + name: http + selector: + app: + +--- +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: + minReplicas: 3 + maxReplicas: 10 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 70 + +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: +spec: + minAvailable: 2 + selector: + matchLabels: + app: diff --git a/devops/orchestration/kubernetes-ops/references/best-practices.md b/devops/orchestration/kubernetes-ops/references/best-practices.md new file mode 100644 index 0000000..dbf6571 --- /dev/null +++ b/devops/orchestration/kubernetes-ops/references/best-practices.md @@ -0,0 +1,187 @@ +# Kubernetes Best Practices + +## Resource Management + +### Always Set Resource Requests and Limits +```yaml +resources: + requests: + memory: "256Mi" + cpu: "250m" + limits: + memory: "512Mi" + cpu: "500m" +``` + +**Guidelines:** +- Requests = guaranteed resources +- Limits = maximum resources +- Set requests based on normal usage +- Set limits based on peak usage +- Memory limit = 2x request is common +- Avoid CPU limits in most cases (causes throttling) + +### Use Horizontal Pod Autoscaler +```yaml +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: myapp-hpa +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: myapp + minReplicas: 2 + maxReplicas: 10 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 70 +``` + +## Pod Configuration + +### Use Liveness and Readiness Probes +```yaml +livenessProbe: + httpGet: + path: /healthz + port: 8080 + initialDelaySeconds: 30 + periodSeconds: 10 + failureThreshold: 3 + +readinessProbe: + httpGet: + path: /ready + port: 8080 + initialDelaySeconds: 5 + periodSeconds: 5 +``` + +### Configure Pod Disruption Budgets +```yaml +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: myapp-pdb +spec: + minAvailable: 2 + selector: + matchLabels: + app: myapp +``` + +### Use Anti-Affinity for High Availability +```yaml +affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchLabels: + app: myapp + topologyKey: kubernetes.io/hostname +``` + +## Security + +### Run as Non-Root +```yaml +securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + fsGroup: 1000 +``` + +### Read-Only Root Filesystem +```yaml +securityContext: + readOnlyRootFilesystem: true +volumeMounts: +- name: tmp + mountPath: /tmp +volumes: +- name: tmp + emptyDir: {} +``` + +### Drop All Capabilities +```yaml +securityContext: + capabilities: + drop: + - ALL +``` + +## Networking + +### Use Network Policies +```yaml +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: deny-all +spec: + podSelector: {} + policyTypes: + - Ingress + - Egress +``` + +### Service Mesh for mTLS +- Istio, Linkerd, or Consul Connect +- Automatic encryption between services +- Traffic management capabilities + +## Configuration Management + +### Use ConfigMaps for Configuration +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: app-config +data: + LOG_LEVEL: "info" + DATABASE_HOST: "postgres.default.svc" +``` + +### Use Secrets for Sensitive Data +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: app-secrets +type: Opaque +stringData: + DATABASE_PASSWORD: "secret123" +``` + +### External Secrets for Production +- Use External Secrets Operator +- Integrate with Vault, AWS Secrets Manager, etc. +- Never commit secrets to git + +## Observability + +### Structured Logging +- Output JSON logs +- Include correlation IDs +- Use consistent field names + +### Metrics +- Expose Prometheus metrics +- Use standard naming conventions +- Include SLI metrics + +### Distributed Tracing +- Implement OpenTelemetry +- Propagate trace context +- Sample appropriately diff --git a/devops/orchestration/kubernetes-ops/references/troubleshooting-guide.md b/devops/orchestration/kubernetes-ops/references/troubleshooting-guide.md new file mode 100644 index 0000000..6b743d4 --- /dev/null +++ b/devops/orchestration/kubernetes-ops/references/troubleshooting-guide.md @@ -0,0 +1,165 @@ +# Kubernetes Troubleshooting Guide + +## Common Issues and Solutions + +### Pod Issues + +#### Pod Stuck in Pending +```bash +# Check events +kubectl describe pod -n + +# Common causes: +# - Insufficient resources +kubectl describe nodes | grep -A 5 "Allocated resources" + +# - No matching nodes (taints/tolerations) +kubectl get nodes -o json | jq '.items[].spec.taints' + +# - PVC not bound +kubectl get pvc -n +``` + +#### Pod in CrashLoopBackOff +```bash +# Check logs +kubectl logs -n --previous + +# Check container exit code +kubectl get pod -n -o jsonpath='{.status.containerStatuses[0].lastState.terminated.exitCode}' + +# Common exit codes: +# 0 - Success (check livenessProbe) +# 1 - Application error +# 137 - OOMKilled (increase memory) +# 139 - Segmentation fault +# 143 - SIGTERM received +``` + +#### Pod in ImagePullBackOff +```bash +# Check image name +kubectl get pod -n -o jsonpath='{.spec.containers[0].image}' + +# Verify image exists +docker pull + +# Check imagePullSecrets +kubectl get pod -n -o jsonpath='{.spec.imagePullSecrets}' +kubectl get secret -n -o jsonpath='{.data.\.dockerconfigjson}' | base64 -d +``` + +### Service Issues + +#### Service Not Accessible +```bash +# Verify endpoints exist +kubectl get endpoints -n + +# Check selector matches pod labels +kubectl get svc -n -o jsonpath='{.spec.selector}' +kubectl get pods -n --show-labels + +# Test from within cluster +kubectl run debug --rm -it --image=busybox -- wget -qO- http://..svc.cluster.local +``` + +#### DNS Resolution Issues +```bash +# Test DNS from pod +kubectl run dns-test --rm -it --image=busybox -- nslookup kubernetes.default + +# Check CoreDNS pods +kubectl get pods -n kube-system -l k8s-app=kube-dns + +# Check CoreDNS logs +kubectl logs -n kube-system -l k8s-app=kube-dns +``` + +### Node Issues + +#### Node NotReady +```bash +# Check node conditions +kubectl describe node | grep -A 20 Conditions + +# Check kubelet status +systemctl status kubelet + +# Check kubelet logs +journalctl -u kubelet -f + +# Common causes: +# - Disk pressure +# - Memory pressure +# - Network issues +# - Container runtime issues +``` + +#### Node Disk Pressure +```bash +# Check disk usage +kubectl describe node | grep -A 3 "Allocated resources" + +# Cleanup unused images +docker system prune -af + +# Check for large logs +du -sh /var/log/containers/* +``` + +### Networking Issues + +#### Pod-to-Pod Communication Fails +```bash +# Test connectivity +kubectl exec -- ping + +# Check network policies +kubectl get networkpolicies -n + +# Verify CNI plugin +kubectl get pods -n kube-system | grep -E "calico|weave|flannel|cilium" +``` + +### Storage Issues + +#### PVC Stuck in Pending +```bash +# Check PVC events +kubectl describe pvc -n + +# Verify StorageClass exists +kubectl get storageclass + +# Check provisioner pods +kubectl get pods -n kube-system | grep provisioner +``` + +## Diagnostic Commands Cheat Sheet + +```bash +# Cluster overview +kubectl cluster-info +kubectl get componentstatuses + +# Resource usage +kubectl top nodes +kubectl top pods -n + +# Events +kubectl get events -n --sort-by='.lastTimestamp' + +# Logs +kubectl logs -f -n +kubectl logs -f -n --all-containers + +# Exec into pod +kubectl exec -it -n -- /bin/sh + +# Port forward +kubectl port-forward 8080:80 -n + +# Copy files +kubectl cp /:/path/to/file ./local-file +``` diff --git a/devops/orchestration/kubernetes-ops/scripts/cluster-health-check.sh b/devops/orchestration/kubernetes-ops/scripts/cluster-health-check.sh new file mode 100644 index 0000000..731fb6b --- /dev/null +++ b/devops/orchestration/kubernetes-ops/scripts/cluster-health-check.sh @@ -0,0 +1,78 @@ +#!/bin/bash +# Kubernetes Cluster Health Check Script +# Usage: ./cluster-health-check.sh [namespace] + +set -euo pipefail + +NAMESPACE="${1:-default}" +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' + +echo "=========================================" +echo "Kubernetes Cluster Health Check" +echo "=========================================" +echo "" + +# Check cluster connectivity +echo -n "Checking cluster connectivity... " +if kubectl cluster-info &>/dev/null; then + echo -e "${GREEN}OK${NC}" +else + echo -e "${RED}FAILED${NC}" + exit 1 +fi + +# Node status +echo "" +echo "Node Status:" +echo "------------" +kubectl get nodes -o wide + +# Check for NotReady nodes +NOT_READY=$(kubectl get nodes --no-headers | grep -v " Ready" | wc -l) +if [ "$NOT_READY" -gt 0 ]; then + echo -e "${RED}WARNING: $NOT_READY node(s) not ready${NC}" +fi + +# Pod status in namespace +echo "" +echo "Pod Status in namespace '$NAMESPACE':" +echo "--------------------------------------" +kubectl get pods -n "$NAMESPACE" -o wide + +# Check for failed pods +FAILED_PODS=$(kubectl get pods -n "$NAMESPACE" --no-headers | grep -E "Error|CrashLoopBackOff|ImagePullBackOff" | wc -l) +if [ "$FAILED_PODS" -gt 0 ]; then + echo -e "${RED}WARNING: $FAILED_PODS pod(s) in error state${NC}" +fi + +# Resource usage +echo "" +echo "Resource Usage:" +echo "---------------" +kubectl top nodes 2>/dev/null || echo "Metrics server not available" + +# Recent events +echo "" +echo "Recent Warning Events:" +echo "----------------------" +kubectl get events -n "$NAMESPACE" --field-selector type=Warning --sort-by='.lastTimestamp' | tail -10 + +# PVC status +echo "" +echo "PersistentVolumeClaim Status:" +echo "-----------------------------" +kubectl get pvc -n "$NAMESPACE" 2>/dev/null || echo "No PVCs found" + +# Service status +echo "" +echo "Services:" +echo "---------" +kubectl get svc -n "$NAMESPACE" + +echo "" +echo "=========================================" +echo "Health check complete" +echo "=========================================" diff --git a/devops/orchestration/kubernetes-ops/scripts/namespace-cleanup.sh b/devops/orchestration/kubernetes-ops/scripts/namespace-cleanup.sh new file mode 100644 index 0000000..84986b3 --- /dev/null +++ b/devops/orchestration/kubernetes-ops/scripts/namespace-cleanup.sh @@ -0,0 +1,75 @@ +#!/bin/bash +# Kubernetes Namespace Cleanup Script +# Removes completed jobs, failed pods, and unused resources +# Usage: ./namespace-cleanup.sh [--dry-run] + +set -euo pipefail + +NAMESPACE="${1:-}" +DRY_RUN="${2:-}" + +if [ -z "$NAMESPACE" ]; then + echo "Usage: $0 [--dry-run]" + exit 1 +fi + +if [ "$DRY_RUN" == "--dry-run" ]; then + echo "DRY RUN MODE - No changes will be made" + DELETE_CMD="echo [DRY RUN] Would delete:" +else + DELETE_CMD="kubectl delete" +fi + +echo "=========================================" +echo "Namespace Cleanup: $NAMESPACE" +echo "=========================================" +echo "" + +# Delete completed jobs +echo "Cleaning up completed Jobs..." +COMPLETED_JOBS=$(kubectl get jobs -n "$NAMESPACE" -o jsonpath='{.items[?(@.status.succeeded==1)].metadata.name}' 2>/dev/null) +if [ -n "$COMPLETED_JOBS" ]; then + for job in $COMPLETED_JOBS; do + $DELETE_CMD job "$job" -n "$NAMESPACE" 2>/dev/null || true + done +else + echo "No completed jobs found" +fi + +# Delete failed pods +echo "" +echo "Cleaning up failed Pods..." +FAILED_PODS=$(kubectl get pods -n "$NAMESPACE" --field-selector status.phase=Failed -o name 2>/dev/null) +if [ -n "$FAILED_PODS" ]; then + for pod in $FAILED_PODS; do + $DELETE_CMD "$pod" -n "$NAMESPACE" 2>/dev/null || true + done +else + echo "No failed pods found" +fi + +# Delete evicted pods +echo "" +echo "Cleaning up evicted Pods..." +EVICTED_PODS=$(kubectl get pods -n "$NAMESPACE" -o json | jq -r '.items[] | select(.status.reason=="Evicted") | .metadata.name' 2>/dev/null) +if [ -n "$EVICTED_PODS" ]; then + for pod in $EVICTED_PODS; do + $DELETE_CMD pod "$pod" -n "$NAMESPACE" 2>/dev/null || true + done +else + echo "No evicted pods found" +fi + +# Delete orphaned ReplicaSets (0 replicas, no owner) +echo "" +echo "Cleaning up orphaned ReplicaSets..." +kubectl get rs -n "$NAMESPACE" -o json | jq -r '.items[] | select(.spec.replicas==0) | .metadata.name' 2>/dev/null | while read rs; do + if [ -n "$rs" ]; then + $DELETE_CMD rs "$rs" -n "$NAMESPACE" 2>/dev/null || true + fi +done + +echo "" +echo "=========================================" +echo "Cleanup complete" +echo "=========================================" diff --git a/devops/orchestration/kubernetes-ops/scripts/pod-debug.sh b/devops/orchestration/kubernetes-ops/scripts/pod-debug.sh new file mode 100644 index 0000000..593378b --- /dev/null +++ b/devops/orchestration/kubernetes-ops/scripts/pod-debug.sh @@ -0,0 +1,71 @@ +#!/bin/bash +# Kubernetes Pod Debugging Script +# Usage: ./pod-debug.sh [namespace] + +set -euo pipefail + +POD_NAME="${1:-}" +NAMESPACE="${2:-default}" + +if [ -z "$POD_NAME" ]; then + echo "Usage: $0 [namespace]" + echo "" + echo "Available pods in namespace '$NAMESPACE':" + kubectl get pods -n "$NAMESPACE" --no-headers | awk '{print " " $1}' + exit 1 +fi + +echo "=========================================" +echo "Debugging Pod: $POD_NAME" +echo "Namespace: $NAMESPACE" +echo "=========================================" +echo "" + +# Pod details +echo "Pod Details:" +echo "------------" +kubectl get pod "$POD_NAME" -n "$NAMESPACE" -o wide + +# Pod describe +echo "" +echo "Pod Description:" +echo "----------------" +kubectl describe pod "$POD_NAME" -n "$NAMESPACE" + +# Container logs +echo "" +echo "Container Logs (last 50 lines):" +echo "--------------------------------" +CONTAINERS=$(kubectl get pod "$POD_NAME" -n "$NAMESPACE" -o jsonpath='{.spec.containers[*].name}') +for CONTAINER in $CONTAINERS; do + echo "" + echo "=== Container: $CONTAINER ===" + kubectl logs "$POD_NAME" -n "$NAMESPACE" -c "$CONTAINER" --tail=50 2>/dev/null || echo "No logs available" +done + +# Previous container logs (if crashed) +echo "" +echo "Previous Container Logs (if any):" +echo "----------------------------------" +for CONTAINER in $CONTAINERS; do + echo "" + echo "=== Container: $CONTAINER (previous) ===" + kubectl logs "$POD_NAME" -n "$NAMESPACE" -c "$CONTAINER" --previous --tail=20 2>/dev/null || echo "No previous logs" +done + +# Resource usage +echo "" +echo "Resource Usage:" +echo "---------------" +kubectl top pod "$POD_NAME" -n "$NAMESPACE" 2>/dev/null || echo "Metrics not available" + +# Events for this pod +echo "" +echo "Related Events:" +echo "---------------" +kubectl get events -n "$NAMESPACE" --field-selector involvedObject.name="$POD_NAME" --sort-by='.lastTimestamp' + +echo "" +echo "=========================================" +echo "Debug information complete" +echo "=========================================" diff --git a/devops/orchestration/kustomize/SKILL.md b/devops/orchestration/kustomize/SKILL.md new file mode 100644 index 0000000..e77ea62 --- /dev/null +++ b/devops/orchestration/kustomize/SKILL.md @@ -0,0 +1,454 @@ +--- +name: kustomize +description: Customize Kubernetes manifests without templating using Kustomize. Create base configurations with environment overlays, manage configuration variants, and patch resources declaratively. Use when managing Kubernetes configurations across multiple environments without Helm. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# Kustomize + +Customize Kubernetes resources declaratively without templating. + +## When to Use This Skill + +Use this skill when: +- Managing Kubernetes configs across environments +- Patching existing manifests without modification +- Creating configuration variants from bases +- Customizing third-party manifests +- Preferring declarative over templating approach + +## Prerequisites + +- kubectl 1.14+ (includes kustomize) +- Or standalone kustomize CLI +- Basic Kubernetes manifest knowledge + +## Directory Structure + +``` +myapp/ +├── base/ +│ ├── kustomization.yaml +│ ├── deployment.yaml +│ ├── service.yaml +│ └── configmap.yaml +└── overlays/ + ├── development/ + │ ├── kustomization.yaml + │ └── replica-patch.yaml + ├── staging/ + │ ├── kustomization.yaml + │ └── namespace.yaml + └── production/ + ├── kustomization.yaml + ├── replica-patch.yaml + └── resource-patch.yaml +``` + +## Base Configuration + +### kustomization.yaml + +```yaml +# base/kustomization.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - deployment.yaml + - service.yaml + - configmap.yaml + +commonLabels: + app: myapp + +commonAnnotations: + managed-by: kustomize +``` + +### Base Resources + +```yaml +# base/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: myapp +spec: + replicas: 1 + selector: + matchLabels: + app: myapp + template: + metadata: + labels: + app: myapp + spec: + containers: + - name: myapp + image: myapp:latest + ports: + - containerPort: 8080 + resources: + requests: + memory: "64Mi" + cpu: "100m" + limits: + memory: "128Mi" + cpu: "200m" +``` + +## Overlays + +### Development Overlay + +```yaml +# overlays/development/kustomization.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../../base + +namespace: myapp-dev + +namePrefix: dev- + +commonLabels: + environment: development + +images: + - name: myapp + newTag: dev-latest +``` + +### Production Overlay + +```yaml +# overlays/production/kustomization.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../../base + +namespace: myapp-prod + +namePrefix: prod- + +commonLabels: + environment: production + +replicas: + - name: myapp + count: 5 + +images: + - name: myapp + newName: registry.example.com/myapp + newTag: v2.0.0 + +patches: + - path: resource-patch.yaml +``` + +## Patching + +### Strategic Merge Patch + +```yaml +# overlays/production/resource-patch.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: myapp +spec: + template: + spec: + containers: + - name: myapp + resources: + requests: + memory: "256Mi" + cpu: "500m" + limits: + memory: "512Mi" + cpu: "1000m" +``` + +### JSON Patch + +```yaml +# kustomization.yaml +patches: + - target: + kind: Deployment + name: myapp + patch: |- + - op: replace + path: /spec/replicas + value: 5 + - op: add + path: /spec/template/spec/containers/0/env + value: + - name: LOG_LEVEL + value: info +``` + +### Inline Patches + +```yaml +# kustomization.yaml +patches: + - patch: |- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: myapp + spec: + replicas: 3 + target: + kind: Deployment + name: myapp +``` + +## Configuration Generation + +### ConfigMap Generator + +```yaml +# kustomization.yaml +configMapGenerator: + - name: myapp-config + literals: + - APP_ENV=production + - LOG_LEVEL=info + files: + - config.yaml + envs: + - config.env + options: + disableNameSuffixHash: false +``` + +### Secret Generator + +```yaml +# kustomization.yaml +secretGenerator: + - name: myapp-secrets + literals: + - api-key=secret123 + files: + - tls.crt + - tls.key + type: kubernetes.io/tls +``` + +## Image Transformations + +```yaml +# kustomization.yaml +images: + # Change tag + - name: myapp + newTag: v2.0.0 + + # Change registry + - name: myapp + newName: registry.example.com/myapp + newTag: v2.0.0 + + # Use digest + - name: myapp + digest: sha256:abc123... +``` + +## Resource Transformations + +### Name Prefix/Suffix + +```yaml +# kustomization.yaml +namePrefix: prod- +nameSuffix: -v2 +``` + +### Namespace + +```yaml +# kustomization.yaml +namespace: production +``` + +### Labels and Annotations + +```yaml +# kustomization.yaml +commonLabels: + app.kubernetes.io/name: myapp + app.kubernetes.io/environment: production + +commonAnnotations: + example.com/owner: team-a +``` + +### Replicas + +```yaml +# kustomization.yaml +replicas: + - name: myapp + count: 5 + - name: worker + count: 3 +``` + +## Components + +```yaml +# components/monitoring/kustomization.yaml +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component + +resources: + - servicemonitor.yaml + +patches: + - patch: |- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: myapp + spec: + template: + metadata: + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "8080" +``` + +```yaml +# overlays/production/kustomization.yaml +components: + - ../../components/monitoring +``` + +## Remote Resources + +```yaml +# kustomization.yaml +resources: + # Remote Git repository + - https://github.com/org/manifests//base?ref=v1.0.0 + + # Remote URL + - https://raw.githubusercontent.com/org/repo/main/deployment.yaml +``` + +## Commands + +```bash +# Build and view output +kubectl kustomize overlays/production + +# Apply to cluster +kubectl apply -k overlays/production + +# Delete resources +kubectl delete -k overlays/production + +# View diff +kubectl diff -k overlays/production + +# Build with standalone kustomize +kustomize build overlays/production + +# Build and apply +kustomize build overlays/production | kubectl apply -f - +``` + +## Helm Chart Integration + +```yaml +# kustomization.yaml +helmCharts: + - name: prometheus + repo: https://prometheus-community.github.io/helm-charts + version: 25.0.0 + releaseName: prometheus + namespace: monitoring + valuesFile: values.yaml + includeCRDs: true +``` + +## Variable Substitution + +```yaml +# kustomization.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - deployment.yaml + +replacements: + - source: + kind: ConfigMap + name: myapp-config + fieldPath: data.APP_VERSION + targets: + - select: + kind: Deployment + name: myapp + fieldPaths: + - spec.template.spec.containers.[name=myapp].image + options: + delimiter: ':' + index: 1 +``` + +## Common Issues + +### Issue: Name Hash Conflicts +**Problem**: Resources not updating when ConfigMap changes +**Solution**: Enable name suffix hash (default) or use replacement + +### Issue: Patch Not Applying +**Problem**: Strategic merge patch doesn't work +**Solution**: Verify resource names match, use JSON patch for complex changes + +### Issue: Remote Resource Fails +**Problem**: Cannot fetch remote resources +**Solution**: Check URL, verify ref/tag exists, ensure network access + +### Issue: Label Selector Mismatch +**Problem**: commonLabels breaks selectors +**Solution**: Use includeSelectors: false or exclude specific resources + +```yaml +commonLabels: + app: myapp +configurations: + - labelExclusions.yaml +``` + +## Best Practices + +- Keep base manifests environment-agnostic +- Use overlays for environment-specific config +- Prefer strategic merge patches for simple changes +- Use components for optional features +- Pin remote resource versions +- Enable ConfigMap/Secret hash suffixes +- Document overlay structure in README +- Test builds before applying + +## Related Skills + +- [kubernetes-ops](../kubernetes-ops/) - K8s fundamentals +- [helm-charts](../helm-charts/) - Helm alternative +- [argocd-gitops](../argocd-gitops/) - GitOps deployment diff --git a/devops/orchestration/openshift/SKILL.md b/devops/orchestration/openshift/SKILL.md new file mode 100644 index 0000000..6dd6d07 --- /dev/null +++ b/devops/orchestration/openshift/SKILL.md @@ -0,0 +1,461 @@ +--- +name: openshift +description: Manage Red Hat OpenShift clusters and deployments. Configure projects, routes, builds, and deploy applications using OpenShift-specific features. Use when working with OpenShift Container Platform or OKD for enterprise Kubernetes. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# OpenShift + +Deploy and manage applications on Red Hat OpenShift Container Platform. + +## When to Use This Skill + +Use this skill when: +- Deploying applications to OpenShift clusters +- Using OpenShift-specific features (Routes, BuildConfigs) +- Managing projects and RBAC in OpenShift +- Implementing S2I (Source-to-Image) builds +- Working with OpenShift Operators + +## Prerequisites + +- OpenShift cluster access +- oc CLI installed +- Basic Kubernetes knowledge + +## CLI Basics + +### Authentication + +```bash +# Login to cluster +oc login https://api.cluster.example.com:6443 -u admin -p password + +# Login with token +oc login --token=sha256~xxxx --server=https://api.cluster.example.com:6443 + +# Check current context +oc whoami +oc whoami --show-server +oc whoami --show-context + +# Logout +oc logout +``` + +### Project Management + +```bash +# Create project (namespace) +oc new-project myapp --display-name="My App" --description="My Application" + +# Switch project +oc project myapp + +# List projects +oc projects + +# Delete project +oc delete project myapp +``` + +## Deploying Applications + +### From Image + +```bash +# Deploy from container image +oc new-app --image=nginx:latest --name=webserver + +# Deploy from Docker Hub +oc new-app docker.io/library/nginx:latest + +# Deploy with environment variables +oc new-app myimage:latest \ + -e DATABASE_URL=postgres://localhost/db \ + -e APP_ENV=production +``` + +### From Source (S2I) + +```bash +# Deploy from Git repository +oc new-app https://github.com/org/myapp.git + +# Specify builder image +oc new-app nodejs:18~https://github.com/org/nodejs-app.git + +# With context directory +oc new-app https://github.com/org/monorepo.git \ + --context-dir=backend \ + --name=backend-api +``` + +### From Template + +```bash +# List available templates +oc get templates -n openshift + +# Deploy from template +oc new-app postgresql-persistent \ + -p POSTGRESQL_USER=user \ + -p POSTGRESQL_PASSWORD=secret \ + -p POSTGRESQL_DATABASE=mydb +``` + +## Routes + +### Creating Routes + +```yaml +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + name: myapp +spec: + host: myapp.apps.cluster.example.com + to: + kind: Service + name: myapp + weight: 100 + port: + targetPort: 8080 + tls: + termination: edge + insecureEdgeTerminationPolicy: Redirect +``` + +```bash +# Create route via CLI +oc expose svc/myapp + +# Create with custom hostname +oc create route edge myapp \ + --service=myapp \ + --hostname=myapp.apps.cluster.example.com + +# Create passthrough route (TLS termination at pod) +oc create route passthrough myapp-secure --service=myapp +``` + +### A/B Testing + +```yaml +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + name: myapp +spec: + to: + kind: Service + name: myapp-v1 + weight: 90 + alternateBackends: + - kind: Service + name: myapp-v2 + weight: 10 +``` + +## Build Configurations + +### BuildConfig + +```yaml +apiVersion: build.openshift.io/v1 +kind: BuildConfig +metadata: + name: myapp +spec: + source: + type: Git + git: + uri: https://github.com/org/myapp.git + ref: main + strategy: + type: Docker + dockerStrategy: + dockerfilePath: Dockerfile + output: + to: + kind: ImageStreamTag + name: myapp:latest + triggers: + - type: ConfigChange + - type: GitHub + github: + secret: webhook-secret +``` + +### S2I Build + +```yaml +apiVersion: build.openshift.io/v1 +kind: BuildConfig +metadata: + name: myapp +spec: + source: + type: Git + git: + uri: https://github.com/org/myapp.git + strategy: + type: Source + sourceStrategy: + from: + kind: ImageStreamTag + namespace: openshift + name: nodejs:18-ubi8 + env: + - name: NPM_RUN + value: start + output: + to: + kind: ImageStreamTag + name: myapp:latest +``` + +### Build Commands + +```bash +# Start build +oc start-build myapp + +# Start build from local source +oc start-build myapp --from-dir=. + +# Follow build logs +oc start-build myapp --follow + +# View build logs +oc logs -f bc/myapp + +# Cancel build +oc cancel-build myapp-1 +``` + +## Image Streams + +```yaml +apiVersion: image.openshift.io/v1 +kind: ImageStream +metadata: + name: myapp +spec: + lookupPolicy: + local: true + tags: + - name: latest + from: + kind: DockerImage + name: registry.example.com/myapp:latest + importPolicy: + scheduled: true +``` + +```bash +# Create image stream +oc create imagestream myapp + +# Import image +oc import-image myapp:latest \ + --from=docker.io/library/nginx:latest \ + --confirm + +# Tag image +oc tag myapp:latest myapp:production +``` + +## Deployment Configs + +```yaml +apiVersion: apps.openshift.io/v1 +kind: DeploymentConfig +metadata: + name: myapp +spec: + replicas: 3 + selector: + app: myapp + template: + metadata: + labels: + app: myapp + spec: + containers: + - name: myapp + image: myapp:latest + ports: + - containerPort: 8080 + resources: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "256Mi" + cpu: "500m" + triggers: + - type: ConfigChange + - type: ImageChange + imageChangeParams: + automatic: true + containerNames: + - myapp + from: + kind: ImageStreamTag + name: myapp:latest + strategy: + type: Rolling + rollingParams: + maxSurge: 25% + maxUnavailable: 25% +``` + +## ConfigMaps and Secrets + +```bash +# Create ConfigMap +oc create configmap myapp-config \ + --from-literal=APP_ENV=production \ + --from-file=config.yaml + +# Create Secret +oc create secret generic myapp-secrets \ + --from-literal=password=secret123 + +# Mount as volume +oc set volume dc/myapp \ + --add --name=config \ + --type=configmap \ + --configmap-name=myapp-config \ + --mount-path=/etc/config + +# Set as environment +oc set env dc/myapp --from=secret/myapp-secrets +``` + +## Security Context Constraints + +```bash +# List SCCs +oc get scc + +# View SCC details +oc describe scc restricted + +# Grant SCC to service account +oc adm policy add-scc-to-user anyuid -z myapp-sa -n myproject + +# Create service account +oc create serviceaccount myapp-sa +``` + +### Custom SCC + +```yaml +apiVersion: security.openshift.io/v1 +kind: SecurityContextConstraints +metadata: + name: myapp-scc +allowPrivilegedContainer: false +runAsUser: + type: MustRunAsNonRoot +seLinuxContext: + type: MustRunAs +fsGroup: + type: RunAsAny +volumes: + - configMap + - secret + - persistentVolumeClaim +users: + - system:serviceaccount:myproject:myapp-sa +``` + +## Operators + +```bash +# List available operators +oc get packagemanifests -n openshift-marketplace + +# Subscribe to operator +cat <= 0.95 + failureLimit: 3 + provider: + prometheus: + address: http://prometheus:9090 + query: | + sum(rate(http_requests_total{service="{{args.service-name}}",status=~"2.*"}[5m])) + / + sum(rate(http_requests_total{service="{{args.service-name}}"}[5m])) +``` + +## Rolling Deployment + +### Kubernetes Default + +```yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: myapp +spec: + replicas: 5 + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 # Max pods above desired + maxUnavailable: 0 # Max pods unavailable + selector: + matchLabels: + app: myapp + template: + metadata: + labels: + app: myapp + spec: + containers: + - name: myapp + image: myapp:v2.0.0 + ports: + - containerPort: 8080 + readinessProbe: + httpGet: + path: /ready + port: 8080 + initialDelaySeconds: 5 + periodSeconds: 5 + livenessProbe: + httpGet: + path: /health + port: 8080 + initialDelaySeconds: 10 + periodSeconds: 10 +``` + +### Rolling Update Commands + +```bash +# Update image +kubectl set image deployment/myapp myapp=myapp:v2.0.0 + +# Watch rollout +kubectl rollout status deployment/myapp + +# Pause rollout +kubectl rollout pause deployment/myapp + +# Resume rollout +kubectl rollout resume deployment/myapp + +# Rollback +kubectl rollout undo deployment/myapp + +# Rollback to specific revision +kubectl rollout undo deployment/myapp --to-revision=2 + +# View history +kubectl rollout history deployment/myapp +``` + +## Health Checks + +### Comprehensive Health Endpoint + +```python +# Flask health endpoint +from flask import Flask, jsonify +import psycopg2 +import redis + +app = Flask(__name__) + +@app.route('/health') +def health(): + """Liveness probe - is the app running?""" + return jsonify({'status': 'healthy'}), 200 + +@app.route('/ready') +def ready(): + """Readiness probe - can the app serve traffic?""" + checks = {} + + # Database check + try: + conn = psycopg2.connect(DATABASE_URL) + conn.close() + checks['database'] = 'ok' + except Exception as e: + checks['database'] = str(e) + return jsonify({'status': 'unhealthy', 'checks': checks}), 503 + + # Redis check + try: + r = redis.from_url(REDIS_URL) + r.ping() + checks['redis'] = 'ok' + except Exception as e: + checks['redis'] = str(e) + return jsonify({'status': 'unhealthy', 'checks': checks}), 503 + + return jsonify({'status': 'healthy', 'checks': checks}), 200 +``` + +## Rollback Procedures + +### Automated Rollback + +```bash +#!/bin/bash +# auto-rollback.sh + +DEPLOYMENT=$1 +THRESHOLD=0.95 +INTERVAL=60 + +echo "Monitoring deployment $DEPLOYMENT" + +while true; do + # Get success rate from Prometheus + SUCCESS_RATE=$(curl -s "http://prometheus:9090/api/v1/query?query=sum(rate(http_requests_total{status=~\"2.*\"}[5m]))/sum(rate(http_requests_total[5m]))" | jq -r '.data.result[0].value[1]') + + echo "Current success rate: $SUCCESS_RATE" + + if (( $(echo "$SUCCESS_RATE < $THRESHOLD" | bc -l) )); then + echo "Success rate below threshold! Rolling back..." + kubectl rollout undo deployment/$DEPLOYMENT + exit 1 + fi + + sleep $INTERVAL +done +``` + +### Manual Rollback Checklist + +```markdown +## Rollback Checklist + +### Before Rollback +- [ ] Confirm issue is deployment-related +- [ ] Document current error rates +- [ ] Notify team in #deployments channel + +### During Rollback +- [ ] Execute rollback command +- [ ] Monitor rollback progress +- [ ] Verify old version is serving traffic + +### After Rollback +- [ ] Confirm error rates normalized +- [ ] Update incident ticket +- [ ] Schedule post-mortem +``` + +## Common Issues + +### Issue: Slow Deployments +**Problem**: Rollout takes too long +**Solution**: Increase maxSurge, decrease minReadySeconds + +### Issue: Failed Health Checks +**Problem**: Pods not becoming ready +**Solution**: Check probe endpoints, increase timeouts + +### Issue: Traffic During Rollback +**Problem**: Errors during switch +**Solution**: Use connection draining, implement graceful shutdown + +## Best Practices + +- Always implement health checks +- Use connection draining +- Test rollback procedures regularly +- Monitor key metrics during deployment +- Implement circuit breakers +- Use deployment slots/environments +- Automate deployment verification +- Document rollback procedures + +## Related Skills + +- [kubernetes-ops](../../orchestration/kubernetes-ops/) - K8s deployment basics +- [argocd-gitops](../../orchestration/argocd-gitops/) - GitOps deployments +- [feature-flags](../feature-flags/) - Progressive rollout diff --git a/devops/release/feature-flags/SKILL.md b/devops/release/feature-flags/SKILL.md new file mode 100644 index 0000000..3f91e77 --- /dev/null +++ b/devops/release/feature-flags/SKILL.md @@ -0,0 +1,472 @@ +--- +name: feature-flags +description: Implement feature flags for progressive feature rollout using LaunchDarkly, Unleash, or custom solutions. Control feature visibility, perform A/B testing, and enable trunk-based development. Use when implementing gradual rollouts, feature toggles, or experimentation platforms. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# Feature Flags + +Control feature releases and enable progressive rollout with feature flag systems. + +## When to Use This Skill + +Use this skill when: +- Implementing gradual feature rollouts +- Enabling trunk-based development +- Running A/B tests and experiments +- Managing feature lifecycles +- Implementing kill switches for production + +## Prerequisites + +- Application code access +- Feature flag service or self-hosted solution +- Basic understanding of deployment patterns + +## Feature Flag Types + +| Type | Purpose | Example | +|------|---------|---------| +| Release | Control feature visibility | New checkout flow | +| Experiment | A/B testing | Button color test | +| Ops | Runtime configuration | Rate limiting | +| Permission | User access control | Premium features | +| Kill Switch | Emergency disable | Third-party integration | + +## LaunchDarkly + +### SDK Setup (Node.js) + +```javascript +const LaunchDarkly = require('launchdarkly-node-server-sdk'); + +const client = LaunchDarkly.init(process.env.LAUNCHDARKLY_SDK_KEY); + +await client.waitForInitialization(); + +// Evaluate flag +const user = { + key: 'user-123', + email: 'user@example.com', + custom: { + plan: 'premium', + company: 'acme' + } +}; + +const showNewFeature = await client.variation('new-checkout', user, false); + +if (showNewFeature) { + // New feature code +} else { + // Existing code +} +``` + +### React SDK + +```javascript +import { withLDProvider, useFlags, useLDClient } from 'launchdarkly-react-client-sdk'; + +// Provider setup +export default withLDProvider({ + clientSideID: 'your-client-side-id', + user: { + key: 'user-123', + email: 'user@example.com' + } +})(App); + +// Using flags in component +function FeatureComponent() { + const { newCheckout, experimentVariant } = useFlags(); + const ldClient = useLDClient(); + + // Track events + const handleClick = () => { + ldClient.track('checkout-started'); + }; + + if (newCheckout) { + return ; + } + return ; +} +``` + +### Targeting Rules + +```yaml +# LaunchDarkly targeting configuration +flag: new-checkout +targeting: + # Individual users + targets: + - variation: true + values: ['user-123', 'user-456'] + + # Rules + rules: + # Beta users + - variation: true + clauses: + - attribute: email + op: endsWith + values: ['@company.com'] + + # Premium plan + - variation: true + clauses: + - attribute: plan + op: in + values: ['premium', 'enterprise'] + + # Percentage rollout + - variation: true + rollout: + variations: + - variation: true + weight: 20000 # 20% + - variation: false + weight: 80000 # 80% + + # Default + fallthrough: + variation: false +``` + +## Unleash + +### Server Setup + +```yaml +# docker-compose.yml +version: '3.8' + +services: + unleash: + image: unleashorg/unleash-server:latest + ports: + - "4242:4242" + environment: + - DATABASE_URL=postgres://postgres:password@db/unleash + - DATABASE_SSL=false + depends_on: + - db + + db: + image: postgres:15 + environment: + - POSTGRES_PASSWORD=password + - POSTGRES_DB=unleash + volumes: + - postgres-data:/var/lib/postgresql/data + +volumes: + postgres-data: +``` + +### SDK Setup (Node.js) + +```javascript +const { initialize } = require('unleash-client'); + +const unleash = initialize({ + url: 'http://localhost:4242/api', + appName: 'my-app', + customHeaders: { + Authorization: 'your-api-token' + } +}); + +unleash.on('ready', () => { + // Check feature + const isEnabled = unleash.isEnabled('new-checkout'); + + // With context + const context = { + userId: 'user-123', + properties: { + plan: 'premium' + } + }; + + const isEnabledForUser = unleash.isEnabled('new-checkout', context); + + // Get variant + const variant = unleash.getVariant('experiment-flag', context); + console.log(variant.name); // 'control' or 'treatment' +}); +``` + +### Activation Strategies + +```yaml +# Standard strategies +strategies: + - name: default + # On/off for everyone + + - name: userWithId + parameters: + userIds: 'user-1,user-2,user-3' + + - name: gradualRolloutUserId + parameters: + percentage: 25 + groupId: 'new-feature' + + - name: gradualRolloutRandom + parameters: + percentage: 50 + + - name: flexibleRollout + parameters: + rollout: 30 + stickiness: userId + groupId: 'checkout-exp' +``` + +## Custom Implementation + +### Database-Backed Flags + +```python +# models.py +from django.db import models + +class FeatureFlag(models.Model): + name = models.CharField(max_length=100, unique=True) + enabled = models.BooleanField(default=False) + rollout_percentage = models.IntegerField(default=0) + allowed_users = models.JSONField(default=list) + rules = models.JSONField(default=dict) + created_at = models.DateTimeField(auto_now_add=True) + updated_at = models.DateTimeField(auto_now=True) + +# service.py +import hashlib + +class FeatureFlagService: + def __init__(self): + self._cache = {} + + def is_enabled(self, flag_name, user_id=None, context=None): + flag = self._get_flag(flag_name) + + if not flag or not flag.enabled: + return False + + # Check user allowlist + if user_id and user_id in flag.allowed_users: + return True + + # Check rules + if context and self._evaluate_rules(flag.rules, context): + return True + + # Check percentage rollout + if flag.rollout_percentage > 0 and user_id: + return self._is_in_rollout(flag_name, user_id, flag.rollout_percentage) + + return flag.rollout_percentage == 100 + + def _is_in_rollout(self, flag_name, user_id, percentage): + hash_input = f"{flag_name}:{user_id}" + hash_value = int(hashlib.md5(hash_input.encode()).hexdigest(), 16) + return (hash_value % 100) < percentage + + def _evaluate_rules(self, rules, context): + for rule in rules.get('rules', []): + if self._evaluate_rule(rule, context): + return True + return False +``` + +### Redis-Backed Flags + +```python +import redis +import json + +class RedisFeatureFlags: + def __init__(self, redis_url): + self.redis = redis.from_url(redis_url) + self.prefix = 'feature_flag:' + + def set_flag(self, name, config): + key = f"{self.prefix}{name}" + self.redis.set(key, json.dumps(config)) + + def is_enabled(self, name, user_id=None): + key = f"{self.prefix}{name}" + data = self.redis.get(key) + + if not data: + return False + + config = json.loads(data) + + if not config.get('enabled', False): + return False + + # User allowlist + if user_id in config.get('users', []): + return True + + # Percentage rollout + percentage = config.get('percentage', 0) + if percentage == 100: + return True + + if percentage > 0 and user_id: + return self._hash_user(name, user_id) < percentage + + return False + + def _hash_user(self, flag, user_id): + import hashlib + hash_input = f"{flag}:{user_id}" + return int(hashlib.sha256(hash_input.encode()).hexdigest(), 16) % 100 +``` + +## Testing with Feature Flags + +### Unit Testing + +```javascript +// Jest mocking +jest.mock('launchdarkly-node-server-sdk', () => ({ + init: jest.fn(() => ({ + waitForInitialization: jest.fn().mockResolvedValue(undefined), + variation: jest.fn() + })) +})); + +describe('Checkout', () => { + it('shows new checkout when flag enabled', async () => { + const ldClient = require('launchdarkly-node-server-sdk').init(); + ldClient.variation.mockResolvedValue(true); + + const result = await renderCheckout(user); + expect(result).toContain('NewCheckout'); + }); + + it('shows old checkout when flag disabled', async () => { + const ldClient = require('launchdarkly-node-server-sdk').init(); + ldClient.variation.mockResolvedValue(false); + + const result = await renderCheckout(user); + expect(result).toContain('OldCheckout'); + }); +}); +``` + +### Integration Testing + +```python +# pytest fixtures +import pytest + +@pytest.fixture +def feature_flags(): + """Provide controllable feature flags for testing.""" + flags = {} + + class TestFlags: + def set(self, name, value): + flags[name] = value + + def is_enabled(self, name, **kwargs): + return flags.get(name, False) + + return TestFlags() + +def test_new_checkout(feature_flags): + feature_flags.set('new-checkout', True) + + response = client.get('/checkout') + assert 'new-checkout-form' in response.content +``` + +## Monitoring and Analytics + +### Flag Usage Tracking + +```javascript +// Track flag evaluations +const flagMetrics = { + evaluations: new Map(), + + track(flagName, variation, user) { + const key = `${flagName}:${variation}`; + const count = this.evaluations.get(key) || 0; + this.evaluations.set(key, count + 1); + + // Send to analytics + analytics.track('feature_flag_evaluated', { + flag: flagName, + variation: variation, + userId: user.key + }); + } +}; +``` + +### Stale Flag Detection + +```python +from datetime import datetime, timedelta + +def detect_stale_flags(): + """Find flags that haven't been evaluated recently.""" + stale_threshold = timedelta(days=30) + now = datetime.utcnow() + + stale_flags = [] + for flag in FeatureFlag.objects.all(): + if flag.last_evaluated: + age = now - flag.last_evaluated + if age > stale_threshold: + stale_flags.append({ + 'name': flag.name, + 'last_evaluated': flag.last_evaluated, + 'age_days': age.days + }) + + return stale_flags +``` + +## Common Issues + +### Issue: Inconsistent Flag Evaluation +**Problem**: Same user sees different variations +**Solution**: Use consistent hashing, check caching strategy + +### Issue: Flag Debt Accumulation +**Problem**: Too many old flags in codebase +**Solution**: Implement flag lifecycle, regular cleanup sprints + +### Issue: Performance Impact +**Problem**: Flag evaluation slowing requests +**Solution**: Use local caching, batch evaluations + +## Best Practices + +- Use consistent naming conventions +- Document flag purpose and owner +- Set expiration dates for temporary flags +- Implement flag lifecycle management +- Use gradual rollouts (not 0→100) +- Monitor flag evaluation metrics +- Clean up old flags regularly +- Test both variations in CI + +## Related Skills + +- [blue-green-deploy](../blue-green-deploy/) - Deployment strategies +- [git-workflow](../git-workflow/) - Trunk-based development +- [alerting-oncall](../../observability/alerting-oncall/) - Monitoring rollouts diff --git a/devops/release/git-workflow/SKILL.md b/devops/release/git-workflow/SKILL.md new file mode 100644 index 0000000..18371b2 --- /dev/null +++ b/devops/release/git-workflow/SKILL.md @@ -0,0 +1,435 @@ +--- +name: git-workflow +description: Implement Git branching strategies, PR workflows, and release management patterns. Configure GitFlow, trunk-based development, or GitHub Flow for team collaboration. Use when establishing version control workflows or improving development team collaboration. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# Git Workflow + +Implement effective branching strategies and pull request workflows for team collaboration. + +## When to Use This Skill + +Use this skill when: +- Establishing team Git workflows +- Implementing branching strategies +- Configuring pull request processes +- Setting up release management +- Improving code review practices + +## Prerequisites + +- Git installed +- Repository hosting (GitHub, GitLab, Bitbucket) +- Basic Git knowledge + +## Branching Strategies + +### Trunk-Based Development + +Best for: Continuous deployment, small teams, mature CI/CD + +``` +main ─────●─────●─────●─────●─────●─────●─────● + │ │ │ │ │ │ + └─● └─● └─● └─● └─● └─● + feature branches (short-lived) +``` + +```bash +# Create short-lived feature branch +git checkout main +git pull origin main +git checkout -b feature/add-login + +# Work and commit frequently +git add . +git commit -m "feat: add login form" + +# Keep branch updated +git fetch origin +git rebase origin/main + +# Merge quickly (same day ideally) +git checkout main +git pull origin main +git merge feature/add-login +git push origin main +git branch -d feature/add-login +``` + +### GitHub Flow + +Best for: Continuous delivery, web applications + +``` +main ─────●─────●───────────●─────────────●─────● + │ ↑ ↑ ↑ + └───●───●───┘ │ │ + feature/login │ │ + │ │ + └───●───●───●───●───────┘ │ + feature/dashboard │ + │ + └───●─────────────────────────┘ + hotfix/security-patch +``` + +```bash +# Create feature branch from main +git checkout main +git pull origin main +git checkout -b feature/user-dashboard + +# Push and create PR +git push -u origin feature/user-dashboard + +# After review, merge via PR (squash recommended) +# Delete branch after merge +``` + +### GitFlow + +Best for: Scheduled releases, versioned products + +``` +main ────────●────────────────●──────────────● + ↑ ↑ ↑ +release ────────┼────●───●──────┼──────────────┼ + │ │ │ │ │ +develop ───●────●────┼───●──●───●───●───●───●──┼ + │ │ │ │ │ │ +feature ───┴─────────┘ │ │ │ │ + │ │ │ │ +hotfix ────────────────────┴───────┼───┼──────┘ + │ │ +feature ────────────────────────────┴───┘ +``` + +```bash +# Initialize GitFlow +git flow init + +# Start feature +git flow feature start user-auth + +# Finish feature (merges to develop) +git flow feature finish user-auth + +# Start release +git flow release start 1.0.0 + +# Finish release (merges to main and develop) +git flow release finish 1.0.0 + +# Hotfix +git flow hotfix start security-fix +git flow hotfix finish security-fix +``` + +## Commit Conventions + +### Conventional Commits + +``` +(): + +[optional body] + +[optional footer(s)] +``` + +Types: +- `feat`: New feature +- `fix`: Bug fix +- `docs`: Documentation +- `style`: Formatting +- `refactor`: Code restructuring +- `test`: Adding tests +- `chore`: Maintenance + +Examples: +```bash +git commit -m "feat(auth): add OAuth2 login support" +git commit -m "fix(api): handle null response from payment service" +git commit -m "docs: update API documentation for v2 endpoints" +git commit -m "refactor(db): optimize user query performance" + +# Breaking change +git commit -m "feat(api)!: change response format for user endpoint + +BREAKING CHANGE: The user endpoint now returns an object instead of array" +``` + +### Commit Message Template + +```bash +# Create template file ~/.gitmessage +# Subject line (50 chars max) + +# Body (72 chars per line max) +# - What changed +# - Why it changed +# - Any side effects + +# Footer +# Fixes #123 +# Co-authored-by: Name + +# Configure Git to use template +git config --global commit.template ~/.gitmessage +``` + +## Pull Request Workflow + +### PR Template + +```markdown + +## Description +Brief description of changes + +## Type of Change +- [ ] Bug fix (non-breaking change) +- [ ] New feature (non-breaking change) +- [ ] Breaking change +- [ ] Documentation update + +## Testing +- [ ] Unit tests added/updated +- [ ] Integration tests added/updated +- [ ] Manual testing performed + +## Checklist +- [ ] Code follows project style guidelines +- [ ] Self-review performed +- [ ] Documentation updated +- [ ] No new warnings introduced + +## Related Issues +Closes # + +## Screenshots (if applicable) +``` + +### Branch Protection Rules + +```yaml +# GitHub branch protection +branch_protection: + branch: main + required_pull_request_reviews: + required_approving_review_count: 1 + dismiss_stale_reviews: true + require_code_owner_reviews: true + required_status_checks: + strict: true + contexts: + - "ci/tests" + - "ci/lint" + restrictions: + users: [] + teams: ["maintainers"] + enforce_admins: true + required_linear_history: true + allow_force_pushes: false + allow_deletions: false +``` + +### Code Owners + +``` +# .github/CODEOWNERS + +# Default owners +* @team-leads + +# Frontend code +/src/frontend/ @frontend-team +*.tsx @frontend-team +*.css @frontend-team + +# Backend code +/src/api/ @backend-team +/src/services/ @backend-team + +# Infrastructure +/terraform/ @platform-team +/k8s/ @platform-team +Dockerfile @platform-team + +# Documentation +/docs/ @tech-writers +*.md @tech-writers +``` + +## Git Hooks + +### Pre-commit Hook + +```bash +#!/bin/sh +# .git/hooks/pre-commit + +# Run linting +npm run lint +if [ $? -ne 0 ]; then + echo "Linting failed. Fix errors before committing." + exit 1 +fi + +# Run tests +npm run test:unit +if [ $? -ne 0 ]; then + echo "Tests failed. Fix tests before committing." + exit 1 +fi + +# Check for debug statements +if grep -r "console.log\|debugger\|binding.pry" --include="*.js" --include="*.ts" --include="*.rb" src/; then + echo "Remove debug statements before committing." + exit 1 +fi +``` + +### Using Husky + +```json +// package.json +{ + "husky": { + "hooks": { + "pre-commit": "lint-staged", + "commit-msg": "commitlint -E HUSKY_GIT_PARAMS" + } + }, + "lint-staged": { + "*.{js,ts,tsx}": ["eslint --fix", "prettier --write"], + "*.{css,scss}": ["prettier --write"] + } +} +``` + +### Commitlint Configuration + +```javascript +// commitlint.config.js +module.exports = { + extends: ['@commitlint/config-conventional'], + rules: { + 'type-enum': [ + 2, + 'always', + ['feat', 'fix', 'docs', 'style', 'refactor', 'test', 'chore', 'revert'] + ], + 'subject-max-length': [2, 'always', 72], + 'body-max-line-length': [2, 'always', 100] + } +}; +``` + +## Release Workflow + +### Automated Release with Tags + +```bash +# Create annotated tag +git tag -a v1.0.0 -m "Release version 1.0.0" + +# Push tag +git push origin v1.0.0 + +# Create release from tag (GitHub CLI) +gh release create v1.0.0 \ + --title "Release 1.0.0" \ + --notes "Release notes here" \ + --target main +``` + +### Changelog Generation + +```bash +# Using conventional-changelog +npx conventional-changelog -p angular -i CHANGELOG.md -s + +# Using git-cliff +git cliff -o CHANGELOG.md +``` + +## Common Git Operations + +### Rebase vs Merge + +```bash +# Rebase (clean history) +git checkout feature/my-feature +git rebase main +git push --force-with-lease + +# Merge (preserve history) +git checkout main +git merge feature/my-feature + +# Squash merge (single commit) +git merge --squash feature/my-feature +git commit -m "feat: add feature X" +``` + +### Cherry Pick + +```bash +# Apply specific commit to current branch +git cherry-pick abc123 + +# Cherry pick range +git cherry-pick abc123..def456 + +# Cherry pick without committing +git cherry-pick -n abc123 +``` + +### Interactive Rebase + +```bash +# Clean up last 3 commits +git rebase -i HEAD~3 + +# In editor: +# pick abc123 First commit +# squash def456 Second commit +# reword ghi789 Third commit +``` + +## Common Issues + +### Issue: Merge Conflicts +**Problem**: Conflicts when merging branches +**Solution**: Rebase frequently, communicate with team, use smaller PRs + +### Issue: Diverged Branches +**Problem**: Local branch far behind remote +**Solution**: `git pull --rebase` or `git fetch && git rebase origin/main` + +### Issue: Accidental Commit to Wrong Branch +**Problem**: Committed to main instead of feature +**Solution**: `git reset HEAD~1`, checkout correct branch, recommit + +## Best Practices + +- Keep branches short-lived (< 1 week) +- Write meaningful commit messages +- Use PR templates consistently +- Require code reviews +- Protect main/master branch +- Automate checks with CI +- Squash merge for clean history +- Delete branches after merge + +## Related Skills + +- [semantic-versioning](../semantic-versioning/) - Version management +- [github-actions](../../ci-cd/github-actions/) - CI/CD automation +- [feature-flags](../feature-flags/) - Feature management diff --git a/devops/release/semantic-versioning/SKILL.md b/devops/release/semantic-versioning/SKILL.md new file mode 100644 index 0000000..a44d856 --- /dev/null +++ b/devops/release/semantic-versioning/SKILL.md @@ -0,0 +1,477 @@ +--- +name: semantic-versioning +description: Automate versioning and changelog generation using semantic versioning principles. Configure release automation, version bumping, and changelog tools. Use when implementing version management or automating release processes. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# Semantic Versioning + +Automate version management and changelog generation following SemVer principles. + +## When to Use This Skill + +Use this skill when: +- Implementing version numbering standards +- Automating release versioning +- Generating changelogs automatically +- Setting up release pipelines +- Managing package versions + +## Prerequisites + +- Git repository with commit history +- Node.js (for most tools) +- Conventional commits (recommended) + +## Semantic Versioning Basics + +### Version Format + +``` +MAJOR.MINOR.PATCH[-PRERELEASE][+BUILD] + +Examples: +1.0.0 +2.1.3 +1.0.0-alpha.1 +1.0.0-beta.2+build.123 +``` + +### Version Components + +| Component | When to Increment | +|-----------|-------------------| +| MAJOR | Breaking changes (incompatible API changes) | +| MINOR | New features (backward compatible) | +| PATCH | Bug fixes (backward compatible) | +| PRERELEASE | Pre-release versions (alpha, beta, rc) | +| BUILD | Build metadata (ignored in precedence) | + +### Version Precedence + +``` +1.0.0-alpha < 1.0.0-alpha.1 < 1.0.0-alpha.beta +< 1.0.0-beta < 1.0.0-beta.2 < 1.0.0-beta.11 +< 1.0.0-rc.1 < 1.0.0 < 2.0.0 +``` + +## Conventional Commits to Version + +```yaml +Commit Type → Version Bump: + feat: → MINOR + fix: → PATCH + docs: → PATCH (or no release) + style: → PATCH (or no release) + refactor: → PATCH + perf: → PATCH + test: → No release + chore: → No release + + BREAKING CHANGE: → MAJOR + feat!: → MAJOR + fix!: → MAJOR +``` + +## semantic-release + +### Installation + +```bash +npm install --save-dev semantic-release \ + @semantic-release/changelog \ + @semantic-release/git +``` + +### Configuration + +```json +// .releaserc.json +{ + "branches": ["main"], + "plugins": [ + "@semantic-release/commit-analyzer", + "@semantic-release/release-notes-generator", + ["@semantic-release/changelog", { + "changelogFile": "CHANGELOG.md" + }], + ["@semantic-release/npm", { + "npmPublish": true + }], + ["@semantic-release/git", { + "assets": ["CHANGELOG.md", "package.json", "package-lock.json"], + "message": "chore(release): ${nextRelease.version} [skip ci]\n\n${nextRelease.notes}" + }], + "@semantic-release/github" + ] +} +``` + +### Advanced Configuration + +```javascript +// release.config.js +module.exports = { + branches: [ + 'main', + { name: 'beta', prerelease: true }, + { name: 'alpha', prerelease: true } + ], + plugins: [ + ['@semantic-release/commit-analyzer', { + preset: 'angular', + releaseRules: [ + { type: 'docs', release: 'patch' }, + { type: 'refactor', release: 'patch' }, + { type: 'style', release: 'patch' }, + { type: 'perf', release: 'patch' }, + { breaking: true, release: 'major' } + ] + }], + ['@semantic-release/release-notes-generator', { + preset: 'angular', + writerOpts: { + commitsSort: ['subject', 'scope'] + } + }], + '@semantic-release/changelog', + '@semantic-release/npm', + '@semantic-release/git', + '@semantic-release/github' + ] +}; +``` + +### GitHub Actions Integration + +```yaml +# .github/workflows/release.yml +name: Release + +on: + push: + branches: [main] + +jobs: + release: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + persist-credentials: false + + - uses: actions/setup-node@v4 + with: + node-version: '20' + + - run: npm ci + + - name: Release + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + NPM_TOKEN: ${{ secrets.NPM_TOKEN }} + run: npx semantic-release +``` + +## standard-version + +### Installation + +```bash +npm install --save-dev standard-version +``` + +### Configuration + +```json +// .versionrc.json +{ + "types": [ + { "type": "feat", "section": "Features" }, + { "type": "fix", "section": "Bug Fixes" }, + { "type": "docs", "section": "Documentation" }, + { "type": "style", "section": "Styling" }, + { "type": "refactor", "section": "Code Refactoring" }, + { "type": "perf", "section": "Performance" }, + { "type": "test", "section": "Tests" }, + { "type": "chore", "section": "Maintenance" } + ], + "skip": { + "bump": false, + "changelog": false, + "commit": false, + "tag": false + }, + "commitUrlFormat": "https://github.com/owner/repo/commit/{{hash}}", + "compareUrlFormat": "https://github.com/owner/repo/compare/{{previousTag}}...{{currentTag}}" +} +``` + +### Usage + +```bash +# First release +npx standard-version --first-release + +# Regular release (auto-detect version bump) +npx standard-version + +# Specific version bump +npx standard-version --release-as minor +npx standard-version --release-as 1.1.0 + +# Pre-release +npx standard-version --prerelease alpha +npx standard-version --prerelease beta + +# Dry run +npx standard-version --dry-run + +# Skip specific steps +npx standard-version --skip.changelog +``` + +### NPM Scripts + +```json +// package.json +{ + "scripts": { + "release": "standard-version", + "release:minor": "standard-version --release-as minor", + "release:major": "standard-version --release-as major", + "release:alpha": "standard-version --prerelease alpha", + "release:beta": "standard-version --prerelease beta", + "release:dry": "standard-version --dry-run" + } +} +``` + +## Changelog Generation + +### conventional-changelog + +```bash +# Install +npm install -g conventional-changelog-cli + +# Generate changelog +conventional-changelog -p angular -i CHANGELOG.md -s + +# Generate all history +conventional-changelog -p angular -i CHANGELOG.md -s -r 0 +``` + +### git-cliff + +```bash +# Install +cargo install git-cliff + +# Generate changelog +git cliff -o CHANGELOG.md +``` + +```toml +# cliff.toml +[changelog] +header = "# Changelog\n\n" +body = """ +{% for group, commits in commits | group_by(attribute="group") %} +## {{ group | upper_first }} +{% for commit in commits %} +- {{ commit.message | upper_first }}\ +{% endfor %} +{% endfor %} +""" +trim = true + +[git] +conventional_commits = true +filter_unconventional = true +commit_preprocessors = [ + { pattern = '\((\w+)\s#([0-9]+)\)', replace = "([#${2}](https://github.com/owner/repo/issues/${2}))" }, +] +commit_parsers = [ + { message = "^feat", group = "Features" }, + { message = "^fix", group = "Bug Fixes" }, + { message = "^doc", group = "Documentation" }, + { message = "^perf", group = "Performance" }, + { message = "^refactor", group = "Refactoring" }, + { message = "^style", group = "Styling" }, + { message = "^test", group = "Testing" }, + { message = "^chore", group = "Miscellaneous" }, +] +filter_commits = true +tag_pattern = "v[0-9]*" +``` + +## Version Bumping Scripts + +### Bash Script + +```bash +#!/bin/bash +# bump-version.sh + +CURRENT_VERSION=$(cat package.json | jq -r '.version') +echo "Current version: $CURRENT_VERSION" + +IFS='.' read -r MAJOR MINOR PATCH <<< "$CURRENT_VERSION" + +case $1 in + major) + NEW_VERSION="$((MAJOR + 1)).0.0" + ;; + minor) + NEW_VERSION="$MAJOR.$((MINOR + 1)).0" + ;; + patch) + NEW_VERSION="$MAJOR.$MINOR.$((PATCH + 1))" + ;; + *) + echo "Usage: $0 {major|minor|patch}" + exit 1 + ;; +esac + +echo "New version: $NEW_VERSION" + +# Update package.json +npm version $NEW_VERSION --no-git-tag-version + +# Create git tag +git add package.json package-lock.json +git commit -m "chore: bump version to $NEW_VERSION" +git tag -a "v$NEW_VERSION" -m "Version $NEW_VERSION" +``` + +### Python Script + +```python +#!/usr/bin/env python3 +# bump_version.py + +import re +import sys +import subprocess + +def get_current_version(): + with open('setup.py', 'r') as f: + content = f.read() + match = re.search(r"version=['\"]([^'\"]+)['\"]", content) + return match.group(1) if match else None + +def bump_version(current, bump_type): + major, minor, patch = map(int, current.split('.')) + + if bump_type == 'major': + return f'{major + 1}.0.0' + elif bump_type == 'minor': + return f'{major}.{minor + 1}.0' + elif bump_type == 'patch': + return f'{major}.{minor}.{patch + 1}' + else: + raise ValueError(f'Invalid bump type: {bump_type}') + +def update_version(old_version, new_version): + with open('setup.py', 'r') as f: + content = f.read() + + content = content.replace(f"version='{old_version}'", f"version='{new_version}'") + + with open('setup.py', 'w') as f: + f.write(content) + +if __name__ == '__main__': + bump_type = sys.argv[1] if len(sys.argv) > 1 else 'patch' + current = get_current_version() + new = bump_version(current, bump_type) + + print(f'Bumping version: {current} → {new}') + update_version(current, new) + + subprocess.run(['git', 'add', 'setup.py']) + subprocess.run(['git', 'commit', '-m', f'chore: bump version to {new}']) + subprocess.run(['git', 'tag', '-a', f'v{new}', '-m', f'Version {new}']) +``` + +## Multi-Package Versioning + +### Lerna + +```json +// lerna.json +{ + "version": "independent", + "npmClient": "npm", + "command": { + "version": { + "conventionalCommits": true, + "message": "chore(release): publish" + }, + "publish": { + "conventionalCommits": true + } + } +} +``` + +```bash +# Version all changed packages +npx lerna version + +# Publish all changed packages +npx lerna publish +``` + +### Changesets + +```bash +# Initialize +npx @changesets/cli init + +# Add changeset +npx changeset add + +# Version packages +npx changeset version + +# Publish +npx changeset publish +``` + +## Common Issues + +### Issue: No Version Bump +**Problem**: semantic-release not creating release +**Solution**: Check commit format, verify branch configuration + +### Issue: Wrong Version Calculated +**Problem**: Major/minor/patch incorrectly determined +**Solution**: Review commit analyzer rules, check for missing prefixes + +### Issue: Duplicate Tags +**Problem**: Tag already exists +**Solution**: Clean up tags, verify version wasn't already released + +## Best Practices + +- Use conventional commits consistently +- Automate version bumping in CI +- Generate changelogs automatically +- Tag releases in Git +- Use pre-release versions for testing +- Document breaking changes clearly +- Include migration guides for major versions +- Lock dependencies with exact versions + +## Related Skills + +- [git-workflow](../git-workflow/) - Branching strategies +- [github-actions](../../ci-cd/github-actions/) - CI automation +- [feature-flags](../feature-flags/) - Progressive rollout diff --git a/infrastructure/cloud-aws/aws-ec2/SKILL.md b/infrastructure/cloud-aws/aws-ec2/SKILL.md new file mode 100644 index 0000000..045cce2 --- /dev/null +++ b/infrastructure/cloud-aws/aws-ec2/SKILL.md @@ -0,0 +1,81 @@ +--- +name: aws-ec2 +description: Manage EC2 instances, AMIs, and auto-scaling groups. Configure security groups, key pairs, and instance types. Use when deploying compute resources on AWS. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# AWS EC2 + +Deploy and manage Amazon EC2 compute instances. + +## Launch Instance + +```bash +aws ec2 run-instances \ + --image-id ami-0abcdef1234567890 \ + --instance-type t3.micro \ + --key-name my-key \ + --security-group-ids sg-12345678 \ + --subnet-id subnet-12345678 \ + --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=web-server}]' +``` + +## Auto Scaling + +```bash +# Create launch template +aws ec2 create-launch-template \ + --launch-template-name web-template \ + --version-description v1 \ + --launch-template-data '{ + "ImageId": "ami-xxx", + "InstanceType": "t3.micro" + }' + +# Create ASG +aws autoscaling create-auto-scaling-group \ + --auto-scaling-group-name web-asg \ + --launch-template LaunchTemplateName=web-template \ + --min-size 2 --max-size 10 --desired-capacity 2 \ + --vpc-zone-identifier "subnet-xxx,subnet-yyy" +``` + +## User Data + +```bash +#!/bin/bash +yum update -y +yum install -y httpd +systemctl start httpd +systemctl enable httpd +``` + +## Instance Management + +```bash +# List instances +aws ec2 describe-instances --filters "Name=tag:Name,Values=web*" + +# Stop/Start +aws ec2 stop-instances --instance-ids i-xxx +aws ec2 start-instances --instance-ids i-xxx + +# Create AMI +aws ec2 create-image --instance-id i-xxx --name "my-ami" +``` + +## Best Practices + +- Use launch templates +- Implement auto-scaling +- Use spot instances for cost savings +- Regular AMI updates +- Instance metadata service v2 + +## Related Skills + +- [terraform-aws](../terraform-aws/) - IaC deployment +- [aws-vpc](../aws-vpc/) - Networking diff --git a/infrastructure/cloud-aws/aws-ec2/references/ec2-operations.md b/infrastructure/cloud-aws/aws-ec2/references/ec2-operations.md new file mode 100644 index 0000000..ed72cc1 --- /dev/null +++ b/infrastructure/cloud-aws/aws-ec2/references/ec2-operations.md @@ -0,0 +1,98 @@ +# EC2 Operations Reference + +## Instance Management + +```bash +# Launch instance +aws ec2 run-instances \ + --image-id ami-12345678 \ + --instance-type t3.micro \ + --key-name mykey \ + --security-group-ids sg-12345678 \ + --subnet-id subnet-12345678 + +# Start/Stop/Terminate +aws ec2 start-instances --instance-ids i-12345678 +aws ec2 stop-instances --instance-ids i-12345678 +aws ec2 terminate-instances --instance-ids i-12345678 + +# Describe instances +aws ec2 describe-instances \ + --filters "Name=tag:Environment,Values=prod" +``` + +## AMI Management + +```bash +# Create AMI from instance +aws ec2 create-image \ + --instance-id i-12345678 \ + --name "MyApp-$(date +%Y%m%d)" + +# Copy AMI to another region +aws ec2 copy-image \ + --source-image-id ami-12345678 \ + --source-region us-east-1 \ + --region us-west-2 \ + --name "MyApp-copy" +``` + +## Instance Types + +| Type | vCPU | Memory | Use Case | +|------|------|--------|----------| +| t3.micro | 2 | 1 GB | Dev/Test | +| t3.small | 2 | 2 GB | Light apps | +| m5.large | 2 | 8 GB | General | +| c5.large | 2 | 4 GB | Compute | +| r5.large | 2 | 16 GB | Memory | + +## User Data + +```bash +#!/bin/bash +yum update -y +yum install -y httpd +systemctl start httpd +systemctl enable httpd +echo "Hello World" > /var/www/html/index.html +``` + +## Instance Metadata + +```bash +# IMDSv2 +TOKEN=$(curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600") +curl -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/ + +# Common endpoints +/latest/meta-data/instance-id +/latest/meta-data/local-ipv4 +/latest/meta-data/public-ipv4 +/latest/meta-data/iam/security-credentials/role-name +``` + +## Terraform + +```hcl +resource "aws_instance" "app" { + ami = data.aws_ami.amazon_linux.id + instance_type = "t3.micro" + + vpc_security_group_ids = [aws_security_group.app.id] + subnet_id = aws_subnet.private.id + + iam_instance_profile = aws_iam_instance_profile.app.name + + user_data = base64encode(file("userdata.sh")) + + root_block_device { + volume_size = 20 + encrypted = true + } + + tags = { + Name = "app-server" + } +} +``` diff --git a/infrastructure/cloud-aws/aws-ecs-fargate/SKILL.md b/infrastructure/cloud-aws/aws-ecs-fargate/SKILL.md new file mode 100644 index 0000000..8c309fc --- /dev/null +++ b/infrastructure/cloud-aws/aws-ecs-fargate/SKILL.md @@ -0,0 +1,88 @@ +--- +name: aws-ecs-fargate +description: Deploy containers on ECS and Fargate. Configure task definitions, services, and load balancing. Use when running containerized workloads on AWS. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# AWS ECS & Fargate + +Run containerized applications on Amazon ECS with Fargate. + +## Task Definition + +```json +{ + "family": "myapp", + "networkMode": "awsvpc", + "requiresCompatibilities": ["FARGATE"], + "cpu": "256", + "memory": "512", + "executionRoleArn": "arn:aws:iam::xxx:role/ecsTaskExecutionRole", + "containerDefinitions": [{ + "name": "myapp", + "image": "xxx.dkr.ecr.region.amazonaws.com/myapp:latest", + "portMappings": [{ + "containerPort": 8080, + "protocol": "tcp" + }], + "logConfiguration": { + "logDriver": "awslogs", + "options": { + "awslogs-group": "/ecs/myapp", + "awslogs-region": "us-east-1", + "awslogs-stream-prefix": "ecs" + } + } + }] +} +``` + +## Create Service + +```bash +aws ecs create-service \ + --cluster my-cluster \ + --service-name myapp \ + --task-definition myapp:1 \ + --desired-count 2 \ + --launch-type FARGATE \ + --network-configuration '{ + "awsvpcConfiguration": { + "subnets": ["subnet-xxx"], + "securityGroups": ["sg-xxx"], + "assignPublicIp": "ENABLED" + } + }' \ + --load-balancers '[{ + "targetGroupArn": "arn:aws:elasticloadbalancing:...", + "containerName": "myapp", + "containerPort": 8080 + }]' +``` + +## Deployment + +```bash +# Update service +aws ecs update-service \ + --cluster my-cluster \ + --service myapp \ + --task-definition myapp:2 \ + --force-new-deployment +``` + +## Best Practices + +- Use ECR for images +- Implement service discovery +- Configure health checks +- Use secrets manager for secrets +- Enable container insights + +## Related Skills + +- [docker-management](../../../devops/containers/docker-management/) - Container basics +- [container-registries](../../../devops/containers/container-registries/) - ECR diff --git a/infrastructure/cloud-aws/aws-iam/SKILL.md b/infrastructure/cloud-aws/aws-iam/SKILL.md new file mode 100644 index 0000000..e170748 --- /dev/null +++ b/infrastructure/cloud-aws/aws-iam/SKILL.md @@ -0,0 +1,97 @@ +--- +name: aws-iam +description: Manage IAM users, roles, and policies. Implement least-privilege access and security best practices. Use when configuring AWS identity and access management. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# AWS IAM + +Manage identity and access in AWS. + +## IAM Policies + +```json +{ + "Version": "2012-10-17", + "Statement": [{ + "Effect": "Allow", + "Action": [ + "s3:GetObject", + "s3:PutObject" + ], + "Resource": "arn:aws:s3:::my-bucket/*" + }] +} +``` + +## Create Role + +```bash +# Create role with trust policy +aws iam create-role \ + --role-name EC2AppRole \ + --assume-role-policy-document '{ + "Version": "2012-10-17", + "Statement": [{ + "Effect": "Allow", + "Principal": {"Service": "ec2.amazonaws.com"}, + "Action": "sts:AssumeRole" + }] + }' + +# Attach policy +aws iam attach-role-policy \ + --role-name EC2AppRole \ + --policy-arn arn:aws:iam::aws:policy/AmazonS3ReadOnlyAccess +``` + +## Service-Linked Roles + +```bash +# For services like ECS, RDS +aws iam create-service-linked-role \ + --aws-service-name ecs.amazonaws.com +``` + +## Best Practices + +```yaml +security_practices: + - Use roles, not long-term credentials + - Implement least privilege + - Enable MFA + - Regular access reviews + - Use IAM Access Analyzer + - Implement SCPs for organizations +``` + +## Policy Conditions + +```json +{ + "Condition": { + "StringEquals": { + "aws:RequestedRegion": "us-east-1" + }, + "Bool": { + "aws:MultiFactorAuthPresent": "true" + } + } +} +``` + +## Best Practices + +- Follow least privilege +- Use IAM roles for applications +- Enable CloudTrail for auditing +- Regular credential rotation +- Use permission boundaries + +## Related Skills + +- [terraform-aws](../terraform-aws/) - IaC deployment +- [access-review](../../../compliance/governance/access-review/) - Access auditing diff --git a/infrastructure/cloud-aws/aws-lambda/SKILL.md b/infrastructure/cloud-aws/aws-lambda/SKILL.md new file mode 100644 index 0000000..13b7773 --- /dev/null +++ b/infrastructure/cloud-aws/aws-lambda/SKILL.md @@ -0,0 +1,84 @@ +--- +name: aws-lambda +description: Build and deploy serverless functions on AWS Lambda. Configure triggers, manage permissions, and optimize performance. Use when implementing serverless applications. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# AWS Lambda + +Build serverless applications with AWS Lambda. + +## Create Function + +```bash +# Create function +aws lambda create-function \ + --function-name myfunction \ + --runtime python3.11 \ + --handler app.handler \ + --role arn:aws:iam::xxx:role/lambda-role \ + --zip-file fileb://function.zip + +# Update code +aws lambda update-function-code \ + --function-name myfunction \ + --zip-file fileb://function.zip +``` + +## Function Code + +```python +# app.py +import json + +def handler(event, context): + return { + 'statusCode': 200, + 'body': json.dumps({'message': 'Hello!'}) + } +``` + +## API Gateway Integration + +```bash +# Create REST API +aws apigateway create-rest-api --name myapi + +# Add Lambda permission +aws lambda add-permission \ + --function-name myfunction \ + --statement-id apigateway \ + --action lambda:InvokeFunction \ + --principal apigateway.amazonaws.com +``` + +## Environment & Configuration + +```bash +# Set environment variables +aws lambda update-function-configuration \ + --function-name myfunction \ + --environment "Variables={DB_HOST=xxx,API_KEY=yyy}" + +# Set memory and timeout +aws lambda update-function-configuration \ + --function-name myfunction \ + --memory-size 256 \ + --timeout 30 +``` + +## Best Practices + +- Minimize cold starts +- Use layers for dependencies +- Implement proper error handling +- Use provisioned concurrency for latency-sensitive functions +- Monitor with CloudWatch + +## Related Skills + +- [terraform-aws](../terraform-aws/) - IaC deployment +- [aws-iam](../aws-iam/) - Execution roles diff --git a/infrastructure/cloud-aws/aws-rds/SKILL.md b/infrastructure/cloud-aws/aws-rds/SKILL.md new file mode 100644 index 0000000..e97d9ec --- /dev/null +++ b/infrastructure/cloud-aws/aws-rds/SKILL.md @@ -0,0 +1,78 @@ +--- +name: aws-rds +description: Provision and manage RDS databases. Configure backups, replication, and security. Use when deploying managed relational databases on AWS. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# AWS RDS + +Deploy managed relational databases with Amazon RDS. + +## Create Database + +```bash +aws rds create-db-instance \ + --db-instance-identifier mydb \ + --db-instance-class db.t3.micro \ + --engine postgres \ + --engine-version 15 \ + --master-username admin \ + --master-user-password secretpassword \ + --allocated-storage 20 \ + --storage-encrypted \ + --vpc-security-group-ids sg-xxx \ + --db-subnet-group-name my-subnet-group \ + --backup-retention-period 7 \ + --multi-az +``` + +## Parameter Groups + +```bash +aws rds create-db-parameter-group \ + --db-parameter-group-name custom-pg \ + --db-parameter-group-family postgres15 \ + --description "Custom PostgreSQL parameters" + +aws rds modify-db-parameter-group \ + --db-parameter-group-name custom-pg \ + --parameters "ParameterName=max_connections,ParameterValue=200,ApplyMethod=pending-reboot" +``` + +## Snapshots & Recovery + +```bash +# Create snapshot +aws rds create-db-snapshot \ + --db-instance-identifier mydb \ + --db-snapshot-identifier mydb-snapshot + +# Restore from snapshot +aws rds restore-db-instance-from-db-snapshot \ + --db-instance-identifier mydb-restored \ + --db-snapshot-identifier mydb-snapshot +``` + +## Read Replicas + +```bash +aws rds create-db-instance-read-replica \ + --db-instance-identifier mydb-replica \ + --source-db-instance-identifier mydb +``` + +## Best Practices + +- Enable Multi-AZ for production +- Use encryption at rest +- Implement automated backups +- Use read replicas for read scaling +- Store credentials in Secrets Manager + +## Related Skills + +- [terraform-aws](../terraform-aws/) - IaC deployment +- [aws-secrets-manager](../../../security/secrets/aws-secrets-manager/) - Credentials diff --git a/infrastructure/cloud-aws/aws-s3/SKILL.md b/infrastructure/cloud-aws/aws-s3/SKILL.md new file mode 100644 index 0000000..4864f71 --- /dev/null +++ b/infrastructure/cloud-aws/aws-s3/SKILL.md @@ -0,0 +1,88 @@ +--- +name: aws-s3 +description: Configure S3 buckets, policies, and lifecycle rules. Implement versioning, replication, and security. Use when managing object storage on AWS. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# AWS S3 + +Manage object storage with Amazon S3. + +## Create Bucket + +```bash +aws s3api create-bucket \ + --bucket my-bucket \ + --region us-east-1 + +# Enable versioning +aws s3api put-bucket-versioning \ + --bucket my-bucket \ + --versioning-configuration Status=Enabled + +# Block public access +aws s3api put-public-access-block \ + --bucket my-bucket \ + --public-access-block-configuration '{ + "BlockPublicAcls": true, + "IgnorePublicAcls": true, + "BlockPublicPolicy": true, + "RestrictPublicBuckets": true + }' +``` + +## Bucket Policy + +```json +{ + "Version": "2012-10-17", + "Statement": [{ + "Sid": "EnforceHTTPS", + "Effect": "Deny", + "Principal": "*", + "Action": "s3:*", + "Resource": [ + "arn:aws:s3:::my-bucket", + "arn:aws:s3:::my-bucket/*" + ], + "Condition": { + "Bool": {"aws:SecureTransport": "false"} + } + }] +} +``` + +## Lifecycle Rules + +```bash +aws s3api put-bucket-lifecycle-configuration \ + --bucket my-bucket \ + --lifecycle-configuration '{ + "Rules": [{ + "ID": "Archive old objects", + "Status": "Enabled", + "Filter": {"Prefix": "logs/"}, + "Transitions": [{ + "Days": 30, + "StorageClass": "GLACIER" + }], + "Expiration": {"Days": 365} + }] + }' +``` + +## Best Practices + +- Enable versioning +- Block public access +- Use encryption (SSE-S3 or SSE-KMS) +- Implement lifecycle policies +- Enable access logging + +## Related Skills + +- [terraform-aws](../terraform-aws/) - IaC deployment +- [aws-iam](../aws-iam/) - Access policies diff --git a/infrastructure/cloud-aws/aws-vpc/SKILL.md b/infrastructure/cloud-aws/aws-vpc/SKILL.md new file mode 100644 index 0000000..5f03094 --- /dev/null +++ b/infrastructure/cloud-aws/aws-vpc/SKILL.md @@ -0,0 +1,84 @@ +--- +name: aws-vpc +description: Design and implement VPCs and networking. Configure subnets, route tables, and security groups. Use when setting up AWS network infrastructure. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# AWS VPC + +Design and manage Virtual Private Cloud networking. + +## Create VPC + +```bash +# Create VPC +aws ec2 create-vpc --cidr-block 10.0.0.0/16 + +# Create subnets +aws ec2 create-subnet \ + --vpc-id vpc-xxx \ + --cidr-block 10.0.1.0/24 \ + --availability-zone us-east-1a + +# Create internet gateway +aws ec2 create-internet-gateway +aws ec2 attach-internet-gateway --vpc-id vpc-xxx --internet-gateway-id igw-xxx +``` + +## Network Architecture + +``` +VPC (10.0.0.0/16) +├── Public Subnets +│ ├── 10.0.1.0/24 (us-east-1a) +│ └── 10.0.2.0/24 (us-east-1b) +├── Private Subnets +│ ├── 10.0.11.0/24 (us-east-1a) +│ └── 10.0.12.0/24 (us-east-1b) +├── Internet Gateway +├── NAT Gateway (in public subnet) +└── Route Tables +``` + +## Security Groups + +```bash +aws ec2 create-security-group \ + --group-name web-sg \ + --description "Web server security group" \ + --vpc-id vpc-xxx + +aws ec2 authorize-security-group-ingress \ + --group-id sg-xxx \ + --protocol tcp \ + --port 443 \ + --cidr 0.0.0.0/0 +``` + +## NAT Gateway + +```bash +# Allocate EIP +aws ec2 allocate-address --domain vpc + +# Create NAT Gateway +aws ec2 create-nat-gateway \ + --subnet-id subnet-public \ + --allocation-id eipalloc-xxx +``` + +## Best Practices + +- Use multiple AZs +- Separate public/private subnets +- Implement VPC Flow Logs +- Use security groups effectively +- Plan CIDR ranges carefully + +## Related Skills + +- [terraform-aws](../terraform-aws/) - IaC deployment +- [firewall-config](../../../security/network/firewall-config/) - Security diff --git a/infrastructure/cloud-aws/cloudformation/SKILL.md b/infrastructure/cloud-aws/cloudformation/SKILL.md new file mode 100644 index 0000000..167369c --- /dev/null +++ b/infrastructure/cloud-aws/cloudformation/SKILL.md @@ -0,0 +1,93 @@ +--- +name: cloudformation +description: Deploy AWS resources with CloudFormation templates. Create stacks, use nested stacks, and implement drift detection. Use when deploying AWS-native IaC. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# CloudFormation + +Deploy AWS infrastructure with native CloudFormation templates. + +## Template Structure + +```yaml +AWSTemplateFormatVersion: '2010-09-09' +Description: Web application stack + +Parameters: + Environment: + Type: String + AllowedValues: [dev, staging, prod] + +Resources: + WebServer: + Type: AWS::EC2::Instance + Properties: + ImageId: !Ref AMI + InstanceType: t3.micro + Tags: + - Key: Name + Value: !Sub '${Environment}-web' + +Outputs: + InstanceId: + Value: !Ref WebServer + Export: + Name: !Sub '${Environment}-WebServerId' +``` + +## Stack Operations + +```bash +# Create stack +aws cloudformation create-stack \ + --stack-name myapp \ + --template-body file://template.yaml \ + --parameters ParameterKey=Environment,ParameterValue=prod + +# Update stack +aws cloudformation update-stack \ + --stack-name myapp \ + --template-body file://template.yaml + +# Delete stack +aws cloudformation delete-stack --stack-name myapp + +# Detect drift +aws cloudformation detect-stack-drift --stack-name myapp +``` + +## Intrinsic Functions + +```yaml +# Reference +!Ref MyResource + +# Get attribute +!GetAtt MyResource.Arn + +# Substitute +!Sub 'arn:aws:s3:::${BucketName}/*' + +# Conditional +!If [CreateProdResources, 't3.large', 't3.micro'] + +# Join +!Join ['-', [!Ref Environment, 'app', 'bucket']] +``` + +## Best Practices + +- Use change sets before updates +- Implement stack policies +- Use nested stacks for modularity +- Enable termination protection +- Use cfn-lint for validation + +## Related Skills + +- [terraform-aws](../terraform-aws/) - Alternative IaC +- [aws-iam](../aws-iam/) - IAM resources diff --git a/infrastructure/cloud-aws/cloudformation/references/cfn-syntax.md b/infrastructure/cloud-aws/cloudformation/references/cfn-syntax.md new file mode 100644 index 0000000..4eddf05 --- /dev/null +++ b/infrastructure/cloud-aws/cloudformation/references/cfn-syntax.md @@ -0,0 +1,123 @@ +# CloudFormation Syntax Reference + +## Template Structure + +```yaml +AWSTemplateFormatVersion: '2010-09-09' +Description: My CloudFormation Template + +Parameters: + Environment: + Type: String + AllowedValues: [dev, staging, prod] + +Mappings: + RegionMap: + us-east-1: + AMI: ami-12345678 + +Conditions: + IsProd: !Equals [!Ref Environment, prod] + +Resources: + MyBucket: + Type: AWS::S3::Bucket + Properties: + BucketName: !Sub '${AWS::StackName}-bucket' + +Outputs: + BucketName: + Value: !Ref MyBucket + Export: + Name: !Sub '${AWS::StackName}-bucket' +``` + +## Intrinsic Functions + +```yaml +# Reference +!Ref MyResource + +# GetAtt +!GetAtt MyResource.Arn + +# Sub (string substitution) +!Sub '${AWS::StackName}-resource' +!Sub + - 'arn:aws:s3:::${Bucket}/*' + - Bucket: !Ref MyBucket + +# Join +!Join ['-', [!Ref Environment, app]] + +# Select +!Select [0, !GetAZs ''] + +# Split +!Split [',', 'a,b,c'] + +# If +!If [IsProd, 3, 1] + +# ImportValue +!ImportValue ExportedValue +``` + +## Common Patterns + +### Cross-Stack References +```yaml +# Stack A - Export +Outputs: + VpcId: + Value: !Ref VPC + Export: + Name: SharedVPC + +# Stack B - Import +Resources: + Subnet: + Type: AWS::EC2::Subnet + Properties: + VpcId: !ImportValue SharedVPC +``` + +### Nested Stacks +```yaml +Resources: + VPCStack: + Type: AWS::CloudFormation::Stack + Properties: + TemplateURL: https://s3.amazonaws.com/bucket/vpc.yaml + Parameters: + Environment: !Ref Environment +``` + +### DependsOn +```yaml +Resources: + MyInstance: + Type: AWS::EC2::Instance + DependsOn: MySecurityGroup +``` + +## CLI Commands + +```bash +# Create stack +aws cloudformation create-stack \ + --stack-name mystack \ + --template-body file://template.yaml \ + --parameters ParameterKey=Environment,ParameterValue=prod + +# Update stack +aws cloudformation update-stack \ + --stack-name mystack \ + --template-body file://template.yaml + +# Delete stack +aws cloudformation delete-stack --stack-name mystack + +# Validate template +aws cloudformation validate-template --template-body file://template.yaml +``` diff --git a/infrastructure/cloud-aws/terraform-aws/SKILL.md b/infrastructure/cloud-aws/terraform-aws/SKILL.md new file mode 100644 index 0000000..f45aa85 --- /dev/null +++ b/infrastructure/cloud-aws/terraform-aws/SKILL.md @@ -0,0 +1,100 @@ +--- +name: terraform-aws +description: Provision AWS infrastructure with Terraform. Create modules, manage state, and implement IaC best practices. Use when deploying AWS resources declaratively. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# Terraform AWS + +Provision and manage AWS infrastructure with Terraform. + +## Provider Configuration + +```hcl +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 5.0" + } + } + + backend "s3" { + bucket = "terraform-state" + key = "prod/terraform.tfstate" + region = "us-east-1" + } +} + +provider "aws" { + region = var.region + + default_tags { + tags = { + Environment = var.environment + ManagedBy = "terraform" + } + } +} +``` + +## Example Resources + +```hcl +resource "aws_vpc" "main" { + cidr_block = "10.0.0.0/16" + enable_dns_hostnames = true + + tags = { Name = "main-vpc" } +} + +resource "aws_instance" "web" { + ami = data.aws_ami.amazon_linux.id + instance_type = "t3.micro" + subnet_id = aws_subnet.public.id + + tags = { Name = "web-server" } +} +``` + +## Modules + +```hcl +module "vpc" { + source = "terraform-aws-modules/vpc/aws" + + name = "my-vpc" + cidr = "10.0.0.0/16" + + azs = ["us-east-1a", "us-east-1b"] + private_subnets = ["10.0.1.0/24", "10.0.2.0/24"] + public_subnets = ["10.0.101.0/24", "10.0.102.0/24"] + + enable_nat_gateway = true +} +``` + +## Commands + +```bash +terraform init +terraform plan -out=plan.tfplan +terraform apply plan.tfplan +terraform destroy +``` + +## Best Practices + +- Use remote state with locking +- Implement module structure +- Use workspaces or separate states per environment +- Pin provider versions +- Use data sources for AMIs + +## Related Skills + +- [aws-vpc](../aws-vpc/) - VPC networking +- [aws-iam](../aws-iam/) - IAM policies diff --git a/infrastructure/cloud-aws/terraform-aws/assets/vpc-module.tf b/infrastructure/cloud-aws/terraform-aws/assets/vpc-module.tf new file mode 100644 index 0000000..38093fb --- /dev/null +++ b/infrastructure/cloud-aws/terraform-aws/assets/vpc-module.tf @@ -0,0 +1,152 @@ +# AWS VPC Module Template +# Production-ready VPC with public and private subnets + +variable "vpc_cidr" { + description = "CIDR block for VPC" + type = string + default = "10.0.0.0/16" +} + +variable "availability_zones" { + description = "Availability zones" + type = list(string) + default = ["us-east-1a", "us-east-1b", "us-east-1c"] +} + +variable "enable_nat_gateway" { + description = "Enable NAT Gateway for private subnets" + type = bool + default = true +} + +locals { + public_subnets = [for i, az in var.availability_zones : cidrsubnet(var.vpc_cidr, 8, i)] + private_subnets = [for i, az in var.availability_zones : cidrsubnet(var.vpc_cidr, 8, i + 10)] +} + +# VPC +resource "aws_vpc" "main" { + cidr_block = var.vpc_cidr + enable_dns_hostnames = true + enable_dns_support = true + + tags = { + Name = "${var.project_name}-vpc" + } +} + +# Internet Gateway +resource "aws_internet_gateway" "main" { + vpc_id = aws_vpc.main.id + + tags = { + Name = "${var.project_name}-igw" + } +} + +# Public Subnets +resource "aws_subnet" "public" { + count = length(var.availability_zones) + vpc_id = aws_vpc.main.id + cidr_block = local.public_subnets[count.index] + availability_zone = var.availability_zones[count.index] + map_public_ip_on_launch = true + + tags = { + Name = "${var.project_name}-public-${var.availability_zones[count.index]}" + "kubernetes.io/role/elb" = "1" + } +} + +# Private Subnets +resource "aws_subnet" "private" { + count = length(var.availability_zones) + vpc_id = aws_vpc.main.id + cidr_block = local.private_subnets[count.index] + availability_zone = var.availability_zones[count.index] + + tags = { + Name = "${var.project_name}-private-${var.availability_zones[count.index]}" + "kubernetes.io/role/internal-elb" = "1" + } +} + +# Elastic IP for NAT Gateway +resource "aws_eip" "nat" { + count = var.enable_nat_gateway ? 1 : 0 + domain = "vpc" + + tags = { + Name = "${var.project_name}-nat-eip" + } +} + +# NAT Gateway +resource "aws_nat_gateway" "main" { + count = var.enable_nat_gateway ? 1 : 0 + allocation_id = aws_eip.nat[0].id + subnet_id = aws_subnet.public[0].id + + tags = { + Name = "${var.project_name}-nat" + } + + depends_on = [aws_internet_gateway.main] +} + +# Public Route Table +resource "aws_route_table" "public" { + vpc_id = aws_vpc.main.id + + route { + cidr_block = "0.0.0.0/0" + gateway_id = aws_internet_gateway.main.id + } + + tags = { + Name = "${var.project_name}-public-rt" + } +} + +# Private Route Table +resource "aws_route_table" "private" { + vpc_id = aws_vpc.main.id + + dynamic "route" { + for_each = var.enable_nat_gateway ? [1] : [] + content { + cidr_block = "0.0.0.0/0" + nat_gateway_id = aws_nat_gateway.main[0].id + } + } + + tags = { + Name = "${var.project_name}-private-rt" + } +} + +# Route Table Associations +resource "aws_route_table_association" "public" { + count = length(var.availability_zones) + subnet_id = aws_subnet.public[count.index].id + route_table_id = aws_route_table.public.id +} + +resource "aws_route_table_association" "private" { + count = length(var.availability_zones) + subnet_id = aws_subnet.private[count.index].id + route_table_id = aws_route_table.private.id +} + +# Outputs +output "vpc_id" { + value = aws_vpc.main.id +} + +output "public_subnet_ids" { + value = aws_subnet.public[*].id +} + +output "private_subnet_ids" { + value = aws_subnet.private[*].id +} diff --git a/infrastructure/cloud-aws/terraform-aws/references/best-practices.md b/infrastructure/cloud-aws/terraform-aws/references/best-practices.md new file mode 100644 index 0000000..8291885 --- /dev/null +++ b/infrastructure/cloud-aws/terraform-aws/references/best-practices.md @@ -0,0 +1,185 @@ +# Terraform AWS Best Practices + +## Project Structure + +``` +project/ +├── main.tf # Main configuration +├── variables.tf # Input variables +├── outputs.tf # Output values +├── locals.tf # Local values +├── data.tf # Data sources +├── versions.tf # Provider versions +├── terraform.tfvars # Variable values (git-ignored) +├── modules/ # Local modules +│ └── vpc/ +│ ├── main.tf +│ ├── variables.tf +│ └── outputs.tf +└── environments/ # Environment configs + ├── dev/ + ├── staging/ + └── prod/ +``` + +## State Management + +### Remote State with S3 +```hcl +terraform { + backend "s3" { + bucket = "company-terraform-state" + key = "project/env/terraform.tfstate" + region = "us-east-1" + encrypt = true + dynamodb_table = "terraform-locks" + } +} +``` + +### State Locking +```hcl +resource "aws_dynamodb_table" "terraform_locks" { + name = "terraform-locks" + billing_mode = "PAY_PER_REQUEST" + hash_key = "LockID" + + attribute { + name = "LockID" + type = "S" + } +} +``` + +## Security Best Practices + +### Use IAM Roles, Not Credentials +```hcl +provider "aws" { + region = "us-east-1" + # No access_key or secret_key - use IAM role or env vars +} +``` + +### Enable Encryption Everywhere +```hcl +resource "aws_s3_bucket_server_side_encryption_configuration" "example" { + bucket = aws_s3_bucket.example.id + + rule { + apply_server_side_encryption_by_default { + sse_algorithm = "aws:kms" + kms_master_key_id = aws_kms_key.example.arn + } + } +} +``` + +### Use Sensitive Variables +```hcl +variable "database_password" { + type = string + sensitive = true +} +``` + +## Tagging Strategy + +```hcl +locals { + common_tags = { + Project = var.project_name + Environment = var.environment + ManagedBy = "terraform" + Owner = var.team + CostCenter = var.cost_center + } +} + +resource "aws_instance" "example" { + # ... configuration ... + + tags = merge(local.common_tags, { + Name = "example-instance" + Role = "web" + }) +} +``` + +## Module Best Practices + +### Version Pinning +```hcl +module "vpc" { + source = "terraform-aws-modules/vpc/aws" + version = "5.0.0" # Pin specific version + + # ... configuration ... +} +``` + +### Variable Validation +```hcl +variable "environment" { + type = string + description = "Environment name" + + validation { + condition = contains(["dev", "staging", "prod"], var.environment) + error_message = "Environment must be dev, staging, or prod." + } +} +``` + +## Workflow + +### Plan Before Apply +```bash +terraform plan -out=tfplan +terraform apply tfplan +``` + +### Use Workspaces or Directories for Environments +```bash +# Workspaces +terraform workspace new prod +terraform workspace select prod + +# Or separate directories (recommended) +cd environments/prod +terraform apply +``` + +## Common Patterns + +### Data Sources for Existing Resources +```hcl +data "aws_vpc" "existing" { + filter { + name = "tag:Name" + values = ["main-vpc"] + } +} + +resource "aws_subnet" "new" { + vpc_id = data.aws_vpc.existing.id + # ... +} +``` + +### Dynamic Blocks +```hcl +resource "aws_security_group" "example" { + # ... + + dynamic "ingress" { + for_each = var.ingress_rules + content { + from_port = ingress.value.from_port + to_port = ingress.value.to_port + protocol = ingress.value.protocol + cidr_blocks = ingress.value.cidr_blocks + } + } +} +``` diff --git a/infrastructure/cloud-aws/terraform-aws/scripts/tf-init.sh b/infrastructure/cloud-aws/terraform-aws/scripts/tf-init.sh new file mode 100644 index 0000000..a05da5b --- /dev/null +++ b/infrastructure/cloud-aws/terraform-aws/scripts/tf-init.sh @@ -0,0 +1,134 @@ +#!/bin/bash +# Terraform AWS Project Initialization Script +# Usage: ./tf-init.sh [region] + +set -euo pipefail + +PROJECT_NAME="${1:-}" +REGION="${2:-us-east-1}" + +if [ -z "$PROJECT_NAME" ]; then + echo "Usage: $0 [region]" + exit 1 +fi + +echo "=========================================" +echo "Terraform AWS Project Setup" +echo "Project: $PROJECT_NAME" +echo "Region: $REGION" +echo "=========================================" +echo "" + +mkdir -p "$PROJECT_NAME" +cd "$PROJECT_NAME" + +# Create main.tf +cat > main.tf << EOF +terraform { + required_version = ">= 1.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 5.0" + } + } + + # Uncomment for remote state + # backend "s3" { + # bucket = "${PROJECT_NAME}-tfstate" + # key = "terraform.tfstate" + # region = "${REGION}" + # encrypt = true + # dynamodb_table = "${PROJECT_NAME}-tflock" + # } +} + +provider "aws" { + region = var.region + + default_tags { + tags = { + Project = var.project_name + Environment = var.environment + ManagedBy = "terraform" + } + } +} +EOF + +# Create variables.tf +cat > variables.tf << EOF +variable "project_name" { + description = "Project name for tagging" + type = string + default = "${PROJECT_NAME}" +} + +variable "environment" { + description = "Environment (dev, staging, prod)" + type = string + default = "dev" +} + +variable "region" { + description = "AWS region" + type = string + default = "${REGION}" +} +EOF + +# Create outputs.tf +cat > outputs.tf << EOF +output "region" { + description = "AWS region" + value = var.region +} +EOF + +# Create terraform.tfvars +cat > terraform.tfvars << EOF +project_name = "${PROJECT_NAME}" +environment = "dev" +region = "${REGION}" +EOF + +# Create .gitignore +cat > .gitignore << EOF +# Terraform +.terraform/ +*.tfstate +*.tfstate.* +*.tfvars.json +crash.log +*.tfplan + +# Keep tfvars template +!terraform.tfvars.example + +# IDE +.idea/ +*.swp +*.swo +.vscode/ +EOF + +# Initialize Terraform +echo "" +echo "Initializing Terraform..." +terraform init + +echo "" +echo "=========================================" +echo "Project created successfully!" +echo "" +echo "Files created:" +ls -la +echo "" +echo "Next steps:" +echo " 1. cd $PROJECT_NAME" +echo " 2. Edit terraform.tfvars" +echo " 3. Add resources to main.tf" +echo " 4. terraform plan" +echo " 5. terraform apply" +echo "=========================================" diff --git a/infrastructure/cloud-azure/arm-templates/SKILL.md b/infrastructure/cloud-azure/arm-templates/SKILL.md new file mode 100644 index 0000000..e3da270 --- /dev/null +++ b/infrastructure/cloud-azure/arm-templates/SKILL.md @@ -0,0 +1,58 @@ +--- +name: arm-templates +description: Deploy Azure resources with ARM templates and Bicep. Create modular deployments and manage dependencies. Use when deploying Azure-native IaC. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# ARM Templates & Bicep + +Deploy Azure infrastructure with ARM templates and Bicep. + +## Bicep Example + +```bicep +param location string = resourceGroup().location +param vmName string + +resource vm 'Microsoft.Compute/virtualMachines@2023-03-01' = { + name: vmName + location: location + properties: { + hardwareProfile: { + vmSize: 'Standard_B2s' + } + osProfile: { + computerName: vmName + adminUsername: 'azureuser' + } + } +} + +output vmId string = vm.id +``` + +## Deployment + +```bash +# Deploy Bicep +az deployment group create \ + --resource-group mygroup \ + --template-file main.bicep \ + --parameters vmName=myvm + +# Deploy ARM +az deployment group create \ + --resource-group mygroup \ + --template-file template.json \ + --parameters @parameters.json +``` + +## Best Practices + +- Use Bicep over JSON ARM +- Implement modules for reusability +- Use parameter files per environment +- Validate before deployment diff --git a/infrastructure/cloud-azure/arm-templates/references/arm-syntax.md b/infrastructure/cloud-azure/arm-templates/references/arm-syntax.md new file mode 100644 index 0000000..1361ec2 --- /dev/null +++ b/infrastructure/cloud-azure/arm-templates/references/arm-syntax.md @@ -0,0 +1,108 @@ +# ARM Template Syntax Reference + +## Template Structure + +```json +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "environment": { + "type": "string", + "allowedValues": ["dev", "staging", "prod"] + } + }, + "variables": { + "storageAccountName": "[concat('storage', uniqueString(resourceGroup().id))]" + }, + "resources": [], + "outputs": {} +} +``` + +## Functions + +```json +// Concatenation +"[concat('prefix-', parameters('name'), '-suffix')]" + +// Unique string +"[uniqueString(resourceGroup().id)]" + +// Resource ID +"[resourceId('Microsoft.Storage/storageAccounts', variables('storageName'))]" + +// Reference (runtime) +"[reference(resourceId('Microsoft.Storage/storageAccounts', variables('storageName'))).primaryEndpoints.blob]" + +// Conditions +"[if(equals(parameters('environment'), 'prod'), 'Standard_GRS', 'Standard_LRS')]" +``` + +## Resource Example + +```json +{ + "type": "Microsoft.Storage/storageAccounts", + "apiVersion": "2021-09-01", + "name": "[variables('storageAccountName')]", + "location": "[resourceGroup().location]", + "sku": { + "name": "[variables('storageSku')]" + }, + "kind": "StorageV2", + "properties": { + "supportsHttpsTrafficOnly": true, + "minimumTlsVersion": "TLS1_2" + } +} +``` + +## Dependencies + +```json +{ + "type": "Microsoft.Web/sites", + "dependsOn": [ + "[resourceId('Microsoft.Web/serverfarms', variables('appServicePlanName'))]" + ] +} +``` + +## Deployment + +```bash +# Create resource group +az group create --name myRG --location eastus + +# Deploy template +az deployment group create \ + --resource-group myRG \ + --template-file template.json \ + --parameters @parameters.json + +# What-if (preview changes) +az deployment group what-if \ + --resource-group myRG \ + --template-file template.json +``` + +## Bicep (Recommended) + +```bicep +param location string = resourceGroup().location +param environment string + +var storageAccountName = 'st${uniqueString(resourceGroup().id)}' + +resource storageAccount 'Microsoft.Storage/storageAccounts@2021-09-01' = { + name: storageAccountName + location: location + sku: { + name: environment == 'prod' ? 'Standard_GRS' : 'Standard_LRS' + } + kind: 'StorageV2' +} + +output storageEndpoint string = storageAccount.properties.primaryEndpoints.blob +``` diff --git a/infrastructure/cloud-azure/azure-aks/SKILL.md b/infrastructure/cloud-azure/azure-aks/SKILL.md new file mode 100644 index 0000000..afe2f89 --- /dev/null +++ b/infrastructure/cloud-azure/azure-aks/SKILL.md @@ -0,0 +1,62 @@ +--- +name: azure-aks +description: Deploy and manage Azure Kubernetes Service clusters. Configure node pools, networking, and integrations. Use when running Kubernetes workloads on Azure. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# Azure Kubernetes Service + +Deploy managed Kubernetes clusters on Azure. + +## Create Cluster + +```bash +az aks create \ + --resource-group mygroup \ + --name myakscluster \ + --node-count 3 \ + --node-vm-size Standard_B2s \ + --enable-managed-identity \ + --generate-ssh-keys + +# Get credentials +az aks get-credentials --resource-group mygroup --name myakscluster +``` + +## Node Pools + +```bash +az aks nodepool add \ + --resource-group mygroup \ + --cluster-name myakscluster \ + --name gpupool \ + --node-count 1 \ + --node-vm-size Standard_NC6 +``` + +## Enable Add-ons + +```bash +# Enable monitoring +az aks enable-addons \ + --resource-group mygroup \ + --name myakscluster \ + --addons monitoring + +# Enable Azure Policy +az aks enable-addons \ + --resource-group mygroup \ + --name myakscluster \ + --addons azure-policy +``` + +## Best Practices + +- Use managed identity +- Enable Azure CNI for networking +- Implement pod identity +- Use node pools for workload isolation +- Enable cluster autoscaler diff --git a/infrastructure/cloud-azure/azure-functions/SKILL.md b/infrastructure/cloud-azure/azure-functions/SKILL.md new file mode 100644 index 0000000..8ceffa5 --- /dev/null +++ b/infrastructure/cloud-azure/azure-functions/SKILL.md @@ -0,0 +1,54 @@ +--- +name: azure-functions +description: Build serverless applications on Azure Functions. Configure triggers, bindings, and deployment. Use when implementing serverless workloads on Azure. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# Azure Functions + +Build serverless applications with Azure Functions. + +## Create Function App + +```bash +az functionapp create \ + --resource-group mygroup \ + --consumption-plan-location eastus \ + --runtime python \ + --runtime-version 3.11 \ + --functions-version 4 \ + --name myfunctionapp \ + --storage-account mystorageaccount +``` + +## Function Code + +```python +import azure.functions as func + +def main(req: func.HttpRequest) -> func.HttpResponse: + return func.HttpResponse("Hello, World!") +``` + +## Deployment + +```bash +# Deploy using Core Tools +func azure functionapp publish myfunctionapp + +# Deploy using ZIP +az functionapp deployment source config-zip \ + --resource-group mygroup \ + --name myfunctionapp \ + --src function.zip +``` + +## Best Practices + +- Use consumption plan for variable workloads +- Implement Durable Functions for orchestration +- Use managed identity for authentication +- Monitor with Application Insights diff --git a/infrastructure/cloud-azure/azure-networking/SKILL.md b/infrastructure/cloud-azure/azure-networking/SKILL.md new file mode 100644 index 0000000..e64762e --- /dev/null +++ b/infrastructure/cloud-azure/azure-networking/SKILL.md @@ -0,0 +1,60 @@ +--- +name: azure-networking +description: Configure Azure VNets, NSGs, and Azure Firewall. Implement hub-spoke topology and private endpoints. Use when designing Azure network infrastructure. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# Azure Networking + +Design and implement Azure network infrastructure. + +## Create VNet + +```bash +az network vnet create \ + --resource-group mygroup \ + --name myvnet \ + --address-prefix 10.0.0.0/16 \ + --subnet-name default \ + --subnet-prefix 10.0.1.0/24 +``` + +## Network Security Group + +```bash +az network nsg create \ + --resource-group mygroup \ + --name mynsg + +az network nsg rule create \ + --resource-group mygroup \ + --nsg-name mynsg \ + --name AllowHTTPS \ + --priority 100 \ + --destination-port-ranges 443 \ + --access Allow +``` + +## Private Endpoint + +```bash +az network private-endpoint create \ + --resource-group mygroup \ + --name myendpoint \ + --vnet-name myvnet \ + --subnet default \ + --private-connection-resource-id /subscriptions/.../sql/... \ + --group-id sqlServer \ + --connection-name myconnection +``` + +## Best Practices + +- Implement hub-spoke topology +- Use NSGs and Azure Firewall +- Enable DDoS protection +- Use private endpoints +- Implement VNet peering diff --git a/infrastructure/cloud-azure/azure-sql/SKILL.md b/infrastructure/cloud-azure/azure-sql/SKILL.md new file mode 100644 index 0000000..23dbe3a --- /dev/null +++ b/infrastructure/cloud-azure/azure-sql/SKILL.md @@ -0,0 +1,58 @@ +--- +name: azure-sql +description: Provision Azure SQL Database and Cosmos DB. Configure security, backups, and replication. Use when deploying managed databases on Azure. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# Azure SQL + +Deploy managed databases on Azure. + +## Create SQL Database + +```bash +# Create server +az sql server create \ + --name myserver \ + --resource-group mygroup \ + --admin-user sqladmin \ + --admin-password SecureP@ss123 + +# Create database +az sql db create \ + --resource-group mygroup \ + --server myserver \ + --name mydb \ + --service-objective S1 +``` + +## Firewall Rules + +```bash +az sql server firewall-rule create \ + --resource-group mygroup \ + --server myserver \ + --name AllowAzure \ + --start-ip-address 0.0.0.0 \ + --end-ip-address 0.0.0.0 +``` + +## Cosmos DB + +```bash +az cosmosdb create \ + --name mycosmosdb \ + --resource-group mygroup \ + --default-consistency-level Session +``` + +## Best Practices + +- Enable transparent data encryption +- Use Azure AD authentication +- Implement geo-replication +- Configure automated backups +- Use private endpoints diff --git a/infrastructure/cloud-azure/azure-vms/SKILL.md b/infrastructure/cloud-azure/azure-vms/SKILL.md new file mode 100644 index 0000000..b2e59b7 --- /dev/null +++ b/infrastructure/cloud-azure/azure-vms/SKILL.md @@ -0,0 +1,45 @@ +--- +name: azure-vms +description: Manage Azure Virtual Machines and scale sets. Configure availability sets and managed disks. Use when deploying compute resources on Azure. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# Azure Virtual Machines + +Deploy and manage Azure VMs and scale sets. + +## Create VM + +```bash +az vm create \ + --resource-group mygroup \ + --name myvm \ + --image Ubuntu2204 \ + --size Standard_B2s \ + --admin-username azureuser \ + --generate-ssh-keys \ + --nsg-rule SSH +``` + +## Scale Sets + +```bash +az vmss create \ + --resource-group mygroup \ + --name myvmss \ + --image Ubuntu2204 \ + --instance-count 2 \ + --vm-sku Standard_B2s \ + --upgrade-policy-mode automatic +``` + +## Best Practices + +- Use managed disks +- Implement availability zones +- Use scale sets for auto-scaling +- Enable Azure Backup +- Use spot instances for cost savings diff --git a/infrastructure/cloud-azure/azure-vms/references/vm-operations.md b/infrastructure/cloud-azure/azure-vms/references/vm-operations.md new file mode 100644 index 0000000..01ef232 --- /dev/null +++ b/infrastructure/cloud-azure/azure-vms/references/vm-operations.md @@ -0,0 +1,99 @@ +# Azure VM Operations Reference + +## VM Management + +```bash +# Create VM +az vm create \ + --resource-group myRG \ + --name myVM \ + --image Ubuntu2204 \ + --admin-username azureuser \ + --generate-ssh-keys \ + --size Standard_B2s + +# Start/Stop/Delete +az vm start --resource-group myRG --name myVM +az vm stop --resource-group myRG --name myVM +az vm deallocate --resource-group myRG --name myVM +az vm delete --resource-group myRG --name myVM --yes + +# List VMs +az vm list --resource-group myRG -o table +az vm list-ip-addresses --resource-group myRG -o table +``` + +## VM Sizes + +| Size | vCPU | Memory | Use Case | +|------|------|--------|----------| +| Standard_B1s | 1 | 1 GB | Dev/Test | +| Standard_B2s | 2 | 4 GB | Light workloads | +| Standard_D2s_v3 | 2 | 8 GB | General | +| Standard_F2s_v2 | 2 | 4 GB | Compute | +| Standard_E2s_v3 | 2 | 16 GB | Memory | + +```bash +# List available sizes +az vm list-sizes --location eastus -o table + +# Resize VM +az vm resize --resource-group myRG --name myVM --size Standard_D4s_v3 +``` + +## Images + +```bash +# List images +az vm image list --output table +az vm image list --publisher Canonical --all --output table + +# Create image from VM +az vm deallocate --resource-group myRG --name myVM +az vm generalize --resource-group myRG --name myVM +az image create --resource-group myRG --name myImage --source myVM +``` + +## Custom Script Extension + +```bash +az vm extension set \ + --resource-group myRG \ + --vm-name myVM \ + --name customScript \ + --publisher Microsoft.Azure.Extensions \ + --settings '{"commandToExecute":"apt-get update && apt-get install -y nginx"}' +``` + +## Terraform + +```hcl +resource "azurerm_linux_virtual_machine" "main" { + name = "myVM" + resource_group_name = azurerm_resource_group.main.name + location = azurerm_resource_group.main.location + size = "Standard_B2s" + admin_username = "azureuser" + + admin_ssh_key { + username = "azureuser" + public_key = file("~/.ssh/id_rsa.pub") + } + + os_disk { + caching = "ReadWrite" + storage_account_type = "Premium_LRS" + } + + source_image_reference { + publisher = "Canonical" + offer = "0001-com-ubuntu-server-jammy" + sku = "22_04-lts" + version = "latest" + } + + network_interface_ids = [ + azurerm_network_interface.main.id, + ] +} +``` diff --git a/infrastructure/cloud-azure/terraform-azure/SKILL.md b/infrastructure/cloud-azure/terraform-azure/SKILL.md new file mode 100644 index 0000000..9143ed7 --- /dev/null +++ b/infrastructure/cloud-azure/terraform-azure/SKILL.md @@ -0,0 +1,59 @@ +--- +name: terraform-azure +description: Provision Azure infrastructure with Terraform. Configure providers, manage state, and deploy resources. Use when implementing IaC for Azure. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# Terraform Azure + +Provision Azure infrastructure with Terraform. + +## Provider Configuration + +```hcl +terraform { + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "~> 3.0" + } + } + backend "azurerm" { + resource_group_name = "tfstate" + storage_account_name = "tfstate12345" + container_name = "tfstate" + key = "prod.terraform.tfstate" + } +} + +provider "azurerm" { + features {} +} +``` + +## Example Resources + +```hcl +resource "azurerm_resource_group" "main" { + name = "myapp-rg" + location = "East US" +} + +resource "azurerm_virtual_network" "main" { + name = "myapp-vnet" + address_space = ["10.0.0.0/16"] + location = azurerm_resource_group.main.location + resource_group_name = azurerm_resource_group.main.name +} +``` + +## Best Practices + +- Use remote state in Azure Storage +- Implement resource naming conventions +- Use data sources for existing resources +- Tag all resources +- Use modules for reusability diff --git a/infrastructure/cloud-azure/terraform-azure/assets/vnet-module.tf b/infrastructure/cloud-azure/terraform-azure/assets/vnet-module.tf new file mode 100644 index 0000000..45d0ea9 --- /dev/null +++ b/infrastructure/cloud-azure/terraform-azure/assets/vnet-module.tf @@ -0,0 +1,80 @@ +# Azure Virtual Network Module Template + +variable "address_space" { + description = "VNet address space" + type = list(string) + default = ["10.0.0.0/16"] +} + +# Virtual Network +resource "azurerm_virtual_network" "main" { + name = "${var.project_name}-${var.environment}-vnet" + location = azurerm_resource_group.main.location + resource_group_name = azurerm_resource_group.main.name + address_space = var.address_space + + tags = local.common_tags +} + +# Public Subnet +resource "azurerm_subnet" "public" { + name = "public-subnet" + resource_group_name = azurerm_resource_group.main.name + virtual_network_name = azurerm_virtual_network.main.name + address_prefixes = ["10.0.1.0/24"] +} + +# Private Subnet +resource "azurerm_subnet" "private" { + name = "private-subnet" + resource_group_name = azurerm_resource_group.main.name + virtual_network_name = azurerm_virtual_network.main.name + address_prefixes = ["10.0.2.0/24"] +} + +# AKS Subnet +resource "azurerm_subnet" "aks" { + name = "aks-subnet" + resource_group_name = azurerm_resource_group.main.name + virtual_network_name = azurerm_virtual_network.main.name + address_prefixes = ["10.0.10.0/23"] +} + +# Network Security Group +resource "azurerm_network_security_group" "main" { + name = "${var.project_name}-${var.environment}-nsg" + location = azurerm_resource_group.main.location + resource_group_name = azurerm_resource_group.main.name + + security_rule { + name = "SSH" + priority = 1001 + direction = "Inbound" + access = "Allow" + protocol = "Tcp" + source_port_range = "*" + destination_port_range = "22" + source_address_prefix = "*" + destination_address_prefix = "*" + } + + tags = local.common_tags +} + +# NSG Association +resource "azurerm_subnet_network_security_group_association" "public" { + subnet_id = azurerm_subnet.public.id + network_security_group_id = azurerm_network_security_group.main.id +} + +output "vnet_id" { + value = azurerm_virtual_network.main.id +} + +output "subnet_ids" { + value = { + public = azurerm_subnet.public.id + private = azurerm_subnet.private.id + aks = azurerm_subnet.aks.id + } +} diff --git a/infrastructure/cloud-azure/terraform-azure/scripts/tf-init-azure.sh b/infrastructure/cloud-azure/terraform-azure/scripts/tf-init-azure.sh new file mode 100644 index 0000000..b3a64ed --- /dev/null +++ b/infrastructure/cloud-azure/terraform-azure/scripts/tf-init-azure.sh @@ -0,0 +1,130 @@ +#!/bin/bash +# Terraform Azure Project Initialization Script +# Usage: ./tf-init-azure.sh [location] + +set -euo pipefail + +PROJECT_NAME="${1:-}" +LOCATION="${2:-eastus}" + +if [ -z "$PROJECT_NAME" ]; then + echo "Usage: $0 [location]" + exit 1 +fi + +echo "=========================================" +echo "Terraform Azure Project Setup" +echo "Project: $PROJECT_NAME" +echo "Location: $LOCATION" +echo "=========================================" +echo "" + +mkdir -p "$PROJECT_NAME" +cd "$PROJECT_NAME" + +# Create main.tf +cat > main.tf << EOF +terraform { + required_version = ">= 1.0" + + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "~> 3.0" + } + } + + # Uncomment for remote state + # backend "azurerm" { + # resource_group_name = "${PROJECT_NAME}-tfstate-rg" + # storage_account_name = "${PROJECT_NAME}tfstate" + # container_name = "tfstate" + # key = "terraform.tfstate" + # } +} + +provider "azurerm" { + features {} +} + +# Resource Group +resource "azurerm_resource_group" "main" { + name = "\${var.project_name}-\${var.environment}-rg" + location = var.location + + tags = local.common_tags +} + +locals { + common_tags = { + Project = var.project_name + Environment = var.environment + ManagedBy = "terraform" + } +} +EOF + +# Create variables.tf +cat > variables.tf << EOF +variable "project_name" { + description = "Project name for resource naming" + type = string + default = "${PROJECT_NAME}" +} + +variable "environment" { + description = "Environment (dev, staging, prod)" + type = string + default = "dev" +} + +variable "location" { + description = "Azure region" + type = string + default = "${LOCATION}" +} +EOF + +# Create outputs.tf +cat > outputs.tf << EOF +output "resource_group_name" { + description = "Resource group name" + value = azurerm_resource_group.main.name +} + +output "location" { + description = "Azure region" + value = azurerm_resource_group.main.location +} +EOF + +# Create terraform.tfvars +cat > terraform.tfvars << EOF +project_name = "${PROJECT_NAME}" +environment = "dev" +location = "${LOCATION}" +EOF + +# Create .gitignore +cat > .gitignore << EOF +.terraform/ +*.tfstate +*.tfstate.* +*.tfvars.json +crash.log +*.tfplan +!terraform.tfvars.example +.idea/ +*.swp +.vscode/ +EOF + +# Initialize Terraform +echo "" +echo "Initializing Terraform..." +terraform init + +echo "" +echo "=========================================" +echo "Project created successfully!" +echo "=========================================" diff --git a/infrastructure/cloud-gcp/gcp-cloud-functions/SKILL.md b/infrastructure/cloud-gcp/gcp-cloud-functions/SKILL.md new file mode 100644 index 0000000..7ddf81a --- /dev/null +++ b/infrastructure/cloud-gcp/gcp-cloud-functions/SKILL.md @@ -0,0 +1,49 @@ +--- +name: gcp-cloud-functions +description: Deploy serverless functions on Google Cloud Functions. Configure triggers and manage deployments. Use when implementing serverless workloads on GCP. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# GCP Cloud Functions + +Build serverless applications with Cloud Functions. + +## Deploy Function + +```bash +# Deploy HTTP function +gcloud functions deploy hello \ + --runtime=python311 \ + --trigger-http \ + --allow-unauthenticated \ + --entry-point=hello_http + +# Deploy Pub/Sub triggered function +gcloud functions deploy process-message \ + --runtime=python311 \ + --trigger-topic=my-topic \ + --entry-point=process +``` + +## Function Code + +```python +# main.py +def hello_http(request): + return 'Hello, World!' + +def process(event, context): + import base64 + data = base64.b64decode(event['data']).decode('utf-8') + print(f"Received: {data}") +``` + +## Best Practices + +- Use 2nd gen functions for better performance +- Implement proper error handling +- Use environment variables for configuration +- Monitor with Cloud Logging diff --git a/infrastructure/cloud-gcp/gcp-cloud-sql/SKILL.md b/infrastructure/cloud-gcp/gcp-cloud-sql/SKILL.md new file mode 100644 index 0000000..de0d4d5 --- /dev/null +++ b/infrastructure/cloud-gcp/gcp-cloud-sql/SKILL.md @@ -0,0 +1,49 @@ +--- +name: gcp-cloud-sql +description: Provision Cloud SQL and Spanner databases. Configure high availability, backups, and security. Use when deploying managed databases on GCP. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# GCP Cloud SQL + +Deploy managed databases on Google Cloud. + +## Create Instance + +```bash +gcloud sql instances create mydb \ + --database-version=POSTGRES_15 \ + --tier=db-f1-micro \ + --region=us-central1 \ + --root-password=secretpassword \ + --storage-auto-increase \ + --backup-start-time=02:00 + +# Create database +gcloud sql databases create myapp --instance=mydb + +# Create user +gcloud sql users create appuser \ + --instance=mydb \ + --password=userpassword +``` + +## High Availability + +```bash +gcloud sql instances create mydb \ + --database-version=POSTGRES_15 \ + --tier=db-custom-2-8192 \ + --region=us-central1 \ + --availability-type=REGIONAL +``` + +## Best Practices + +- Enable automated backups +- Use Cloud SQL Proxy for connections +- Implement private IP +- Use read replicas for scaling diff --git a/infrastructure/cloud-gcp/gcp-compute/SKILL.md b/infrastructure/cloud-gcp/gcp-compute/SKILL.md new file mode 100644 index 0000000..dbaa485 --- /dev/null +++ b/infrastructure/cloud-gcp/gcp-compute/SKILL.md @@ -0,0 +1,42 @@ +--- +name: gcp-compute +description: Manage Compute Engine instances and instance templates. Configure managed instance groups and preemptible VMs. Use when deploying compute resources on GCP. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# GCP Compute Engine + +Deploy and manage Compute Engine instances. + +## Create Instance + +```bash +gcloud compute instances create web-server \ + --machine-type=e2-medium \ + --zone=us-central1-a \ + --image-family=debian-11 \ + --image-project=debian-cloud \ + --boot-disk-size=20GB \ + --tags=http-server + +# Create from instance template +gcloud compute instance-templates create web-template \ + --machine-type=e2-medium \ + --image-family=debian-11 \ + --image-project=debian-cloud + +gcloud compute instance-groups managed create web-group \ + --template=web-template \ + --size=3 \ + --zone=us-central1-a +``` + +## Best Practices + +- Use managed instance groups +- Implement preemptible VMs for cost savings +- Use custom images for consistency +- Enable shielded VMs diff --git a/infrastructure/cloud-gcp/gcp-gke/SKILL.md b/infrastructure/cloud-gcp/gcp-gke/SKILL.md new file mode 100644 index 0000000..608bcc4 --- /dev/null +++ b/infrastructure/cloud-gcp/gcp-gke/SKILL.md @@ -0,0 +1,55 @@ +--- +name: gcp-gke +description: Deploy and manage Google Kubernetes Engine clusters. Configure node pools, networking, and workload identity. Use when running Kubernetes on GCP. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# Google Kubernetes Engine + +Deploy managed Kubernetes clusters on GCP. + +## Create Cluster + +```bash +gcloud container clusters create my-cluster \ + --num-nodes=3 \ + --machine-type=e2-medium \ + --zone=us-central1-a \ + --enable-autoscaling \ + --min-nodes=1 \ + --max-nodes=5 \ + --workload-pool=${PROJECT_ID}.svc.id.goog + +# Get credentials +gcloud container clusters get-credentials my-cluster --zone=us-central1-a +``` + +## Node Pools + +```bash +gcloud container node-pools create gpu-pool \ + --cluster=my-cluster \ + --zone=us-central1-a \ + --machine-type=n1-standard-4 \ + --accelerator=type=nvidia-tesla-k80,count=1 \ + --num-nodes=1 +``` + +## Workload Identity + +```bash +gcloud iam service-accounts add-iam-policy-binding \ + --role=roles/iam.workloadIdentityUser \ + --member="serviceAccount:${PROJECT_ID}.svc.id.goog[NAMESPACE/KSA_NAME]" \ + GSA_NAME@${PROJECT_ID}.iam.gserviceaccount.com +``` + +## Best Practices + +- Use Workload Identity +- Enable VPC-native clusters +- Implement node auto-provisioning +- Use regional clusters for HA diff --git a/infrastructure/cloud-gcp/gcp-networking/SKILL.md b/infrastructure/cloud-gcp/gcp-networking/SKILL.md new file mode 100644 index 0000000..3c5943c --- /dev/null +++ b/infrastructure/cloud-gcp/gcp-networking/SKILL.md @@ -0,0 +1,59 @@ +--- +name: gcp-networking +description: Configure VPCs, firewall rules, and Cloud NAT. Implement shared VPC and private service connect. Use when designing GCP network infrastructure. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# GCP Networking + +Design and implement GCP network infrastructure. + +## Create VPC + +```bash +gcloud compute networks create my-vpc --subnet-mode=custom + +gcloud compute networks subnets create my-subnet \ + --network=my-vpc \ + --region=us-central1 \ + --range=10.0.0.0/24 +``` + +## Firewall Rules + +```bash +gcloud compute firewall-rules create allow-http \ + --network=my-vpc \ + --allow=tcp:80,tcp:443 \ + --source-ranges=0.0.0.0/0 \ + --target-tags=http-server + +gcloud compute firewall-rules create allow-internal \ + --network=my-vpc \ + --allow=tcp,udp,icmp \ + --source-ranges=10.0.0.0/8 +``` + +## Cloud NAT + +```bash +gcloud compute routers create my-router \ + --network=my-vpc \ + --region=us-central1 + +gcloud compute routers nats create my-nat \ + --router=my-router \ + --region=us-central1 \ + --nat-all-subnet-ip-ranges \ + --auto-allocate-nat-external-ips +``` + +## Best Practices + +- Use Shared VPC for multi-project +- Implement Cloud Armor for DDoS +- Use Private Google Access +- Enable VPC Flow Logs diff --git a/infrastructure/cloud-gcp/terraform-gcp/SKILL.md b/infrastructure/cloud-gcp/terraform-gcp/SKILL.md new file mode 100644 index 0000000..11c7d26 --- /dev/null +++ b/infrastructure/cloud-gcp/terraform-gcp/SKILL.md @@ -0,0 +1,66 @@ +--- +name: terraform-gcp +description: Provision GCP infrastructure with Terraform. Configure providers and deploy Google Cloud resources. Use when implementing IaC for GCP. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# Terraform GCP + +Provision Google Cloud infrastructure with Terraform. + +## Provider Configuration + +```hcl +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = "~> 5.0" + } + } + backend "gcs" { + bucket = "tf-state-bucket" + prefix = "terraform/state" + } +} + +provider "google" { + project = var.project_id + region = var.region +} +``` + +## Example Resources + +```hcl +resource "google_compute_network" "vpc" { + name = "main-vpc" + auto_create_subnetworks = false +} + +resource "google_compute_instance" "vm" { + name = "web-server" + machine_type = "e2-micro" + zone = "us-central1-a" + + boot_disk { + initialize_params { + image = "debian-cloud/debian-11" + } + } + + network_interface { + network = google_compute_network.vpc.name + } +} +``` + +## Best Practices + +- Use service accounts for authentication +- Store state in GCS +- Use labels consistently +- Implement least-privilege IAM diff --git a/infrastructure/cloud-gcp/terraform-gcp/assets/vpc-module.tf b/infrastructure/cloud-gcp/terraform-gcp/assets/vpc-module.tf new file mode 100644 index 0000000..af92b74 --- /dev/null +++ b/infrastructure/cloud-gcp/terraform-gcp/assets/vpc-module.tf @@ -0,0 +1,114 @@ +# GCP VPC Module Template + +variable "network_name" { + description = "VPC network name" + type = string + default = "main" +} + +# VPC Network +resource "google_compute_network" "main" { + name = "${var.project_name}-${var.environment}-vpc" + auto_create_subnetworks = false + project = var.project_id +} + +# Public Subnet +resource "google_compute_subnetwork" "public" { + name = "${var.project_name}-${var.environment}-public" + ip_cidr_range = "10.0.1.0/24" + region = var.region + network = google_compute_network.main.id + project = var.project_id + + secondary_ip_range { + range_name = "gke-pods" + ip_cidr_range = "10.1.0.0/16" + } + + secondary_ip_range { + range_name = "gke-services" + ip_cidr_range = "10.2.0.0/20" + } +} + +# Private Subnet +resource "google_compute_subnetwork" "private" { + name = "${var.project_name}-${var.environment}-private" + ip_cidr_range = "10.0.2.0/24" + region = var.region + network = google_compute_network.main.id + project = var.project_id + private_ip_google_access = true +} + +# Cloud Router (for NAT) +resource "google_compute_router" "main" { + name = "${var.project_name}-${var.environment}-router" + region = var.region + network = google_compute_network.main.id + project = var.project_id +} + +# Cloud NAT +resource "google_compute_router_nat" "main" { + name = "${var.project_name}-${var.environment}-nat" + router = google_compute_router.main.name + region = var.region + project = var.project_id + nat_ip_allocate_option = "AUTO_ONLY" + source_subnetwork_ip_ranges_to_nat = "ALL_SUBNETWORKS_ALL_IP_RANGES" +} + +# Firewall - Allow SSH +resource "google_compute_firewall" "allow_ssh" { + name = "${var.project_name}-${var.environment}-allow-ssh" + network = google_compute_network.main.name + project = var.project_id + + allow { + protocol = "tcp" + ports = ["22"] + } + + source_ranges = ["0.0.0.0/0"] + target_tags = ["ssh"] +} + +# Firewall - Allow Internal +resource "google_compute_firewall" "allow_internal" { + name = "${var.project_name}-${var.environment}-allow-internal" + network = google_compute_network.main.name + project = var.project_id + + allow { + protocol = "icmp" + } + + allow { + protocol = "tcp" + ports = ["0-65535"] + } + + allow { + protocol = "udp" + ports = ["0-65535"] + } + + source_ranges = ["10.0.0.0/8"] +} + +output "network_name" { + value = google_compute_network.main.name +} + +output "network_id" { + value = google_compute_network.main.id +} + +output "subnet_ids" { + value = { + public = google_compute_subnetwork.public.id + private = google_compute_subnetwork.private.id + } +} diff --git a/infrastructure/cloud-gcp/terraform-gcp/scripts/tf-init-gcp.sh b/infrastructure/cloud-gcp/terraform-gcp/scripts/tf-init-gcp.sh new file mode 100644 index 0000000..5a31b1e --- /dev/null +++ b/infrastructure/cloud-gcp/terraform-gcp/scripts/tf-init-gcp.sh @@ -0,0 +1,130 @@ +#!/bin/bash +# Terraform GCP Project Initialization Script +# Usage: ./tf-init-gcp.sh [region] + +set -euo pipefail + +PROJECT_NAME="${1:-}" +GCP_PROJECT="${2:-}" +REGION="${3:-us-central1}" + +if [ -z "$PROJECT_NAME" ] || [ -z "$GCP_PROJECT" ]; then + echo "Usage: $0 [region]" + exit 1 +fi + +echo "=========================================" +echo "Terraform GCP Project Setup" +echo "Project: $PROJECT_NAME" +echo "GCP Project: $GCP_PROJECT" +echo "Region: $REGION" +echo "=========================================" +echo "" + +mkdir -p "$PROJECT_NAME" +cd "$PROJECT_NAME" + +# Create main.tf +cat > main.tf << EOF +terraform { + required_version = ">= 1.0" + + required_providers { + google = { + source = "hashicorp/google" + version = "~> 5.0" + } + } + + # Uncomment for remote state + # backend "gcs" { + # bucket = "${PROJECT_NAME}-tfstate" + # prefix = "terraform/state" + # } +} + +provider "google" { + project = var.project_id + region = var.region +} + +locals { + common_labels = { + project = var.project_name + environment = var.environment + managed-by = "terraform" + } +} +EOF + +# Create variables.tf +cat > variables.tf << EOF +variable "project_name" { + description = "Project name for resource naming" + type = string + default = "${PROJECT_NAME}" +} + +variable "project_id" { + description = "GCP Project ID" + type = string + default = "${GCP_PROJECT}" +} + +variable "environment" { + description = "Environment (dev, staging, prod)" + type = string + default = "dev" +} + +variable "region" { + description = "GCP region" + type = string + default = "${REGION}" +} +EOF + +# Create outputs.tf +cat > outputs.tf << EOF +output "project_id" { + description = "GCP Project ID" + value = var.project_id +} + +output "region" { + description = "GCP region" + value = var.region +} +EOF + +# Create terraform.tfvars +cat > terraform.tfvars << EOF +project_name = "${PROJECT_NAME}" +project_id = "${GCP_PROJECT}" +environment = "dev" +region = "${REGION}" +EOF + +# Create .gitignore +cat > .gitignore << EOF +.terraform/ +*.tfstate +*.tfstate.* +*.tfvars.json +crash.log +*.tfplan +!terraform.tfvars.example +.idea/ +*.swp +.vscode/ +EOF + +# Initialize Terraform +echo "" +echo "Initializing Terraform..." +terraform init + +echo "" +echo "=========================================" +echo "Project created successfully!" +echo "=========================================" diff --git a/infrastructure/databases/database-backups/SKILL.md b/infrastructure/databases/database-backups/SKILL.md new file mode 100644 index 0000000..1c12739 --- /dev/null +++ b/infrastructure/databases/database-backups/SKILL.md @@ -0,0 +1,70 @@ +--- +name: database-backups +description: Implement database backup strategies. Configure automated backups, retention, and recovery testing. Use when designing backup and recovery procedures. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# Database Backups + +Implement comprehensive database backup strategies. + +## Backup Types + +```yaml +backup_types: + full: + description: Complete database copy + frequency: Weekly + + incremental: + description: Changes since last backup + frequency: Daily + + transaction_log: + description: Continuous transaction logging + frequency: Continuous +``` + +## Automated Backup Script + +```bash +#!/bin/bash +DATE=$(date +%Y%m%d_%H%M%S) +BACKUP_DIR="/backups" + +# PostgreSQL +pg_dump -Fc mydb > $BACKUP_DIR/pg_$DATE.dump + +# MySQL +mysqldump -u root -p$MYSQL_PWD mydb | gzip > $BACKUP_DIR/mysql_$DATE.sql.gz + +# Upload to S3 +aws s3 cp $BACKUP_DIR/pg_$DATE.dump s3://backups/postgres/ + +# Cleanup old backups (keep 7 days) +find $BACKUP_DIR -name "*.dump" -mtime +7 -delete +``` + +## Recovery Testing + +```bash +# Create test environment +docker run -d --name restore-test postgres:15 + +# Restore backup +pg_restore -d testdb backup.dump + +# Verify data integrity +psql testdb -c "SELECT COUNT(*) FROM users;" +``` + +## Best Practices + +- 3-2-1 Rule: 3 copies, 2 media types, 1 offsite +- Regular recovery testing +- Encrypt backups at rest +- Monitor backup success +- Document recovery procedures diff --git a/infrastructure/databases/mongodb/SKILL.md b/infrastructure/databases/mongodb/SKILL.md new file mode 100644 index 0000000..aaa0597 --- /dev/null +++ b/infrastructure/databases/mongodb/SKILL.md @@ -0,0 +1,79 @@ +--- +name: mongodb +description: Administer MongoDB databases. Configure replica sets, sharding, and backups. Use when managing MongoDB deployments. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# MongoDB + +Administer MongoDB NoSQL databases. + +## Installation & Setup + +```bash +# Install +apt install mongodb-org + +# Start service +systemctl start mongod + +# Connect +mongosh + +# Create user +use admin +db.createUser({ + user: "admin", + pwd: "secret", + roles: ["root"] +}) +``` + +## Basic Operations + +```javascript +// Create database and collection +use mydb +db.users.insertOne({ name: "John", email: "john@example.com" }) + +// Query +db.users.find({ name: "John" }) +db.users.find().sort({ name: 1 }).limit(10) + +// Index +db.users.createIndex({ email: 1 }, { unique: true }) +``` + +## Replica Set + +```javascript +// Initialize replica set +rs.initiate({ + _id: "myReplicaSet", + members: [ + { _id: 0, host: "mongo1:27017" }, + { _id: 1, host: "mongo2:27017" }, + { _id: 2, host: "mongo3:27017" } + ] +}) +``` + +## Backup + +```bash +# Backup +mongodump --out /backup/ + +# Restore +mongorestore /backup/ +``` + +## Best Practices + +- Use replica sets in production +- Implement proper indexing +- Enable authentication +- Regular backups with mongodump diff --git a/infrastructure/databases/mysql/SKILL.md b/infrastructure/databases/mysql/SKILL.md new file mode 100644 index 0000000..fd89d79 --- /dev/null +++ b/infrastructure/databases/mysql/SKILL.md @@ -0,0 +1,78 @@ +--- +name: mysql +description: Administer MySQL/MariaDB databases. Configure replication and optimize performance. Use when managing MySQL deployments. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# MySQL / MariaDB + +Administer MySQL and MariaDB databases. + +## Installation & Setup + +```bash +# Install +apt install mysql-server + +# Secure installation +mysql_secure_installation + +# Access +mysql -u root -p + +# Create database and user +CREATE DATABASE mydb; +CREATE USER 'myapp'@'%' IDENTIFIED BY 'secret'; +GRANT ALL PRIVILEGES ON mydb.* TO 'myapp'@'%'; +FLUSH PRIVILEGES; +``` + +## Configuration + +```bash +# /etc/mysql/mysql.conf.d/mysqld.cnf +[mysqld] +innodb_buffer_pool_size = 1G +max_connections = 200 +slow_query_log = 1 +long_query_time = 2 +``` + +## Backup & Restore + +```bash +# Backup +mysqldump -u root -p mydb > backup.sql +mysqldump -u root -p --all-databases > full_backup.sql + +# Restore +mysql -u root -p mydb < backup.sql +``` + +## Replication + +```bash +# Primary +[mysqld] +server-id = 1 +log_bin = mysql-bin + +# Replica +CHANGE MASTER TO + MASTER_HOST='primary', + MASTER_USER='replicator', + MASTER_PASSWORD='secret', + MASTER_LOG_FILE='mysql-bin.000001', + MASTER_LOG_POS=0; +START SLAVE; +``` + +## Best Practices + +- Enable slow query logging +- Use InnoDB storage engine +- Regular backups with mysqldump +- Monitor with SHOW PROCESSLIST diff --git a/infrastructure/databases/postgresql/SKILL.md b/infrastructure/databases/postgresql/SKILL.md new file mode 100644 index 0000000..ce80d06 --- /dev/null +++ b/infrastructure/databases/postgresql/SKILL.md @@ -0,0 +1,68 @@ +--- +name: postgresql +description: Administer PostgreSQL databases. Configure replication, backups, and performance tuning. Use when managing PostgreSQL deployments. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# PostgreSQL + +Administer and optimize PostgreSQL databases. + +## Installation & Setup + +```bash +# Install +apt install postgresql postgresql-contrib + +# Access +sudo -u postgres psql + +# Create database and user +CREATE USER myapp WITH PASSWORD 'secret'; +CREATE DATABASE mydb OWNER myapp; +GRANT ALL PRIVILEGES ON DATABASE mydb TO myapp; +``` + +## Configuration + +```bash +# /etc/postgresql/15/main/postgresql.conf +max_connections = 200 +shared_buffers = 256MB +effective_cache_size = 768MB +work_mem = 4MB +maintenance_work_mem = 64MB +``` + +## Backup & Restore + +```bash +# Backup +pg_dump mydb > backup.sql +pg_dump -Fc mydb > backup.dump # Custom format + +# Restore +psql mydb < backup.sql +pg_restore -d mydb backup.dump +``` + +## Replication + +```bash +# Primary +ALTER SYSTEM SET wal_level = replica; +CREATE USER replicator REPLICATION LOGIN PASSWORD 'secret'; + +# Replica +pg_basebackup -h primary -U replicator -D /var/lib/postgresql/15/main -P +``` + +## Best Practices + +- Regular VACUUM and ANALYZE +- Monitor slow queries +- Implement connection pooling (PgBouncer) +- Regular backups with pg_dump or pg_basebackup diff --git a/infrastructure/databases/redis/SKILL.md b/infrastructure/databases/redis/SKILL.md new file mode 100644 index 0000000..f2a8456 --- /dev/null +++ b/infrastructure/databases/redis/SKILL.md @@ -0,0 +1,74 @@ +--- +name: redis +description: Configure Redis for caching and data storage. Set up clustering, persistence, and Sentinel. Use when implementing Redis caching or queues. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# Redis + +Configure Redis for caching and data storage. + +## Installation & Setup + +```bash +# Install +apt install redis-server + +# Configuration +# /etc/redis/redis.conf +bind 0.0.0.0 +protected-mode yes +requirepass yourpassword +maxmemory 256mb +maxmemory-policy allkeys-lru +``` + +## Basic Operations + +```bash +redis-cli -a yourpassword + +# String operations +SET key "value" +GET key +SETEX key 3600 "value" # With TTL + +# Hash +HSET user:1 name "John" email "john@example.com" +HGETALL user:1 + +# List +LPUSH queue "task1" +RPOP queue +``` + +## Persistence + +```bash +# RDB (snapshot) +save 900 1 +save 300 10 + +# AOF (append-only file) +appendonly yes +appendfsync everysec +``` + +## Sentinel (HA) + +```bash +# sentinel.conf +sentinel monitor mymaster 10.0.0.1 6379 2 +sentinel down-after-milliseconds mymaster 30000 +sentinel failover-timeout mymaster 180000 +``` + +## Best Practices + +- Set maxmemory and eviction policy +- Use persistence for critical data +- Implement Sentinel for HA +- Monitor memory usage diff --git a/infrastructure/networking/cdn-setup/SKILL.md b/infrastructure/networking/cdn-setup/SKILL.md new file mode 100644 index 0000000..b943952 --- /dev/null +++ b/infrastructure/networking/cdn-setup/SKILL.md @@ -0,0 +1,59 @@ +--- +name: cdn-setup +description: Configure CDNs for content delivery. Set up CloudFront, Cloudflare, and Fastly. Use when optimizing global content delivery. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# CDN Setup + +Configure content delivery networks. + +## AWS CloudFront + +```bash +aws cloudfront create-distribution --distribution-config '{ + "CallerReference": "my-distribution", + "Origins": { + "Quantity": 1, + "Items": [{ + "Id": "myS3Origin", + "DomainName": "mybucket.s3.amazonaws.com", + "S3OriginConfig": {"OriginAccessIdentity": ""} + }] + }, + "DefaultCacheBehavior": { + "TargetOriginId": "myS3Origin", + "ViewerProtocolPolicy": "redirect-to-https", + "CachePolicyId": "658327ea-f89d-4fab-a63d-7e88639e58f6" + }, + "Enabled": true +}' +``` + +## Cloudflare + +```bash +# Via API +curl -X POST "https://api.cloudflare.com/client/v4/zones" \ + -H "Authorization: Bearer $TOKEN" \ + -d '{"name":"example.com","jump_start":true}' +``` + +## Cache Headers + +```nginx +location ~* \.(jpg|jpeg|png|gif|ico|css|js)$ { + expires 30d; + add_header Cache-Control "public, immutable"; +} +``` + +## Best Practices + +- Set appropriate cache headers +- Use cache invalidation sparingly +- Implement cache warming +- Monitor cache hit ratios diff --git a/infrastructure/networking/dns-management/SKILL.md b/infrastructure/networking/dns-management/SKILL.md new file mode 100644 index 0000000..feb7b3c --- /dev/null +++ b/infrastructure/networking/dns-management/SKILL.md @@ -0,0 +1,67 @@ +--- +name: dns-management +description: Configure DNS zones and records. Manage Route53, Cloud DNS, and self-hosted DNS. Use when setting up DNS infrastructure. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# DNS Management + +Configure and manage DNS infrastructure. + +## AWS Route 53 + +```bash +# Create hosted zone +aws route53 create-hosted-zone --name example.com --caller-reference $(date +%s) + +# Create record +aws route53 change-resource-record-sets --hosted-zone-id ZXXXXX --change-batch '{ + "Changes": [{ + "Action": "CREATE", + "ResourceRecordSet": { + "Name": "www.example.com", + "Type": "A", + "TTL": 300, + "ResourceRecords": [{"Value": "1.2.3.4"}] + } + }] +}' +``` + +## BIND Configuration + +```bash +# /etc/bind/zones/example.com.db +$TTL 86400 +@ IN SOA ns1.example.com. admin.example.com. ( + 2024010101 ; Serial + 3600 ; Refresh + 1800 ; Retry + 604800 ; Expire + 86400 ) ; Minimum TTL + + IN NS ns1.example.com. + IN A 1.2.3.4 +www IN A 1.2.3.4 +``` + +## Common Records + +``` +A - IPv4 address +AAAA - IPv6 address +CNAME - Alias to another domain +MX - Mail server +TXT - Text record (SPF, DKIM) +NS - Name server +``` + +## Best Practices + +- Low TTL during migrations +- Implement DNSSEC +- Use multiple name servers +- Monitor DNS resolution diff --git a/infrastructure/networking/load-balancing/SKILL.md b/infrastructure/networking/load-balancing/SKILL.md new file mode 100644 index 0000000..041344b --- /dev/null +++ b/infrastructure/networking/load-balancing/SKILL.md @@ -0,0 +1,64 @@ +--- +name: load-balancing +description: Configure load balancers and traffic distribution. Implement health checks and SSL termination. Use when distributing traffic across servers. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# Load Balancing + +Distribute traffic across application servers. + +## nginx Load Balancer + +```nginx +upstream backend { + least_conn; + server backend1:8080 weight=3; + server backend2:8080; + server backend3:8080 backup; +} + +server { + listen 80; + + location / { + proxy_pass http://backend; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + } +} +``` + +## HAProxy + +``` +frontend http_front + bind *:80 + default_backend http_back + +backend http_back + balance roundrobin + option httpchk GET /health + server web1 10.0.0.1:8080 check + server web2 10.0.0.2:8080 check +``` + +## AWS ALB + +```bash +aws elbv2 create-load-balancer \ + --name my-alb \ + --subnets subnet-xxx subnet-yyy \ + --security-groups sg-xxx \ + --type application +``` + +## Best Practices + +- Implement health checks +- Use sticky sessions when needed +- Enable connection draining +- Monitor backend health diff --git a/infrastructure/networking/reverse-proxy/SKILL.md b/infrastructure/networking/reverse-proxy/SKILL.md new file mode 100644 index 0000000..57ae702 --- /dev/null +++ b/infrastructure/networking/reverse-proxy/SKILL.md @@ -0,0 +1,68 @@ +--- +name: reverse-proxy +description: Configure nginx and Traefik as reverse proxies. Implement SSL termination and routing. Use when setting up application gateways. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# Reverse Proxy + +Configure reverse proxies for application routing. + +## nginx + +```nginx +server { + listen 80; + server_name api.example.com; + return 301 https://$server_name$request_uri; +} + +server { + listen 443 ssl http2; + server_name api.example.com; + + ssl_certificate /etc/ssl/certs/api.crt; + ssl_certificate_key /etc/ssl/private/api.key; + + location / { + proxy_pass http://backend:8080; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + + location /ws { + proxy_pass http://backend:8080; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + } +} +``` + +## Traefik + +```yaml +# traefik.yml +entryPoints: + web: + address: ":80" + websecure: + address: ":443" + +providers: + docker: + exposedByDefault: false +``` + +## Best Practices + +- Implement SSL termination +- Set proper headers +- Configure timeouts +- Enable gzip compression diff --git a/infrastructure/networking/service-mesh/SKILL.md b/infrastructure/networking/service-mesh/SKILL.md new file mode 100644 index 0000000..ba05693 --- /dev/null +++ b/infrastructure/networking/service-mesh/SKILL.md @@ -0,0 +1,70 @@ +--- +name: service-mesh +description: Implement Istio and Linkerd service meshes. Configure mTLS, traffic management, and observability. Use when managing microservices communication. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# Service Mesh + +Implement service-to-service communication management. + +## Istio Installation + +```bash +istioctl install --set profile=demo + +# Enable sidecar injection +kubectl label namespace default istio-injection=enabled +``` + +## Traffic Management + +```yaml +apiVersion: networking.istio.io/v1alpha3 +kind: VirtualService +metadata: + name: myapp +spec: + hosts: + - myapp + http: + - match: + - headers: + canary: + exact: "true" + route: + - destination: + host: myapp + subset: canary + - route: + - destination: + host: myapp + subset: stable + weight: 90 + - destination: + host: myapp + subset: canary + weight: 10 +``` + +## mTLS + +```yaml +apiVersion: security.istio.io/v1beta1 +kind: PeerAuthentication +metadata: + name: default +spec: + mtls: + mode: STRICT +``` + +## Best Practices + +- Enable strict mTLS +- Implement circuit breakers +- Use traffic shifting for deployments +- Monitor with Kiali and Jaeger diff --git a/infrastructure/servers/linux-administration/SKILL.md b/infrastructure/servers/linux-administration/SKILL.md new file mode 100644 index 0000000..23eba9c --- /dev/null +++ b/infrastructure/servers/linux-administration/SKILL.md @@ -0,0 +1,65 @@ +--- +name: linux-administration +description: System administration for Linux servers. Manage packages, services, and system configuration. Use when administering Linux systems. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# Linux Administration + +Core Linux system administration skills. + +## Package Management + +```bash +# Debian/Ubuntu +apt update && apt upgrade -y +apt install nginx +apt remove nginx +apt autoremove + +# RHEL/CentOS +dnf update +dnf install nginx +dnf remove nginx +``` + +## System Information + +```bash +uname -a # Kernel info +hostnamectl # System info +lscpu # CPU info +free -h # Memory usage +df -h # Disk usage +ip addr # Network interfaces +``` + +## Log Management + +```bash +journalctl -u nginx # Service logs +journalctl -f # Follow logs +tail -f /var/log/syslog # System logs +dmesg # Kernel messages +``` + +## Process Management + +```bash +ps aux | grep nginx +top / htop +kill -9 +pgrep nginx +pkill nginx +``` + +## Best Practices + +- Regular updates +- Minimal installed packages +- Proper file permissions +- Log rotation configuration +- Automated backups diff --git a/infrastructure/servers/performance-tuning/SKILL.md b/infrastructure/servers/performance-tuning/SKILL.md new file mode 100644 index 0000000..4d54b75 --- /dev/null +++ b/infrastructure/servers/performance-tuning/SKILL.md @@ -0,0 +1,69 @@ +--- +name: performance-tuning +description: Optimize Linux system performance. Configure kernel parameters, analyze bottlenecks, and tune resources. Use when improving system performance. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# Performance Tuning + +Optimize Linux system performance. + +## System Monitoring + +```bash +top / htop # Process monitoring +vmstat 1 # Memory statistics +iostat -x 1 # Disk I/O +sar -n DEV 1 # Network statistics +perf top # CPU profiling +``` + +## Kernel Parameters + +```bash +# /etc/sysctl.d/99-performance.conf +vm.swappiness = 10 +net.core.somaxconn = 65535 +net.ipv4.tcp_max_syn_backlog = 65535 +fs.file-max = 2097152 +vm.dirty_ratio = 40 +vm.dirty_background_ratio = 10 +``` + +## File Descriptor Limits + +```bash +# /etc/security/limits.conf +* soft nofile 65535 +* hard nofile 65535 +* soft nproc 65535 +* hard nproc 65535 +``` + +## Disk I/O + +```bash +# Change scheduler +echo noop > /sys/block/sda/queue/scheduler + +# Enable trim for SSDs +fstrim -av +``` + +## Network Tuning + +```bash +# Increase buffers +sysctl -w net.core.rmem_max=134217728 +sysctl -w net.core.wmem_max=134217728 +``` + +## Best Practices + +- Profile before optimizing +- Change one parameter at a time +- Monitor impact of changes +- Document all tuning diff --git a/infrastructure/servers/ssh-configuration/SKILL.md b/infrastructure/servers/ssh-configuration/SKILL.md new file mode 100644 index 0000000..3aa90d9 --- /dev/null +++ b/infrastructure/servers/ssh-configuration/SKILL.md @@ -0,0 +1,76 @@ +--- +name: ssh-configuration +description: Configure SSH servers and clients securely. Manage keys, tunnels, and config files. Use when setting up secure remote access. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# SSH Configuration + +Secure SSH server and client configuration. + +## Key Management + +```bash +# Generate key +ssh-keygen -t ed25519 -C "user@example.com" + +# Copy to server +ssh-copy-id user@server + +# Add to agent +eval "$(ssh-agent -s)" +ssh-add ~/.ssh/id_ed25519 +``` + +## SSH Config (~/.ssh/config) + +``` +Host production + HostName prod.example.com + User deploy + IdentityFile ~/.ssh/prod_key + Port 22 + +Host bastion + HostName bastion.example.com + User admin + +Host internal + HostName 10.0.0.5 + User admin + ProxyJump bastion +``` + +## Secure Server Config + +```bash +# /etc/ssh/sshd_config +PermitRootLogin no +PasswordAuthentication no +PubkeyAuthentication yes +MaxAuthTries 3 +AllowUsers deploy admin +``` + +## Tunneling + +```bash +# Local port forward +ssh -L 8080:internal:80 bastion + +# Remote port forward +ssh -R 8080:localhost:80 server + +# SOCKS proxy +ssh -D 1080 server +``` + +## Best Practices + +- Use ed25519 keys +- Disable password auth +- Use SSH agent forwarding carefully +- Implement jump hosts/bastions diff --git a/infrastructure/servers/systemd-services/SKILL.md b/infrastructure/servers/systemd-services/SKILL.md new file mode 100644 index 0000000..08947f2 --- /dev/null +++ b/infrastructure/servers/systemd-services/SKILL.md @@ -0,0 +1,76 @@ +--- +name: systemd-services +description: Create and manage systemd services and timers. Configure service dependencies and resource limits. Use when managing system services. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# Systemd Services + +Manage system services with systemd. + +## Service Unit File + +```ini +# /etc/systemd/system/myapp.service +[Unit] +Description=My Application +After=network.target + +[Service] +Type=simple +User=myapp +WorkingDirectory=/opt/myapp +ExecStart=/opt/myapp/bin/start +ExecStop=/opt/myapp/bin/stop +Restart=always +RestartSec=5 +Environment=NODE_ENV=production + +[Install] +WantedBy=multi-user.target +``` + +## Service Management + +```bash +systemctl daemon-reload +systemctl start myapp +systemctl stop myapp +systemctl restart myapp +systemctl enable myapp +systemctl status myapp +journalctl -u myapp -f +``` + +## Timer (Cron Replacement) + +```ini +# /etc/systemd/system/backup.timer +[Unit] +Description=Daily backup + +[Timer] +OnCalendar=daily +Persistent=true + +[Install] +WantedBy=timers.target +``` + +## Resource Limits + +```ini +[Service] +MemoryLimit=512M +CPUQuota=50% +``` + +## Best Practices + +- Use Type=notify for better tracking +- Implement proper restart policies +- Use timers instead of cron +- Set resource limits diff --git a/infrastructure/servers/user-management/SKILL.md b/infrastructure/servers/user-management/SKILL.md new file mode 100644 index 0000000..69b5f4d --- /dev/null +++ b/infrastructure/servers/user-management/SKILL.md @@ -0,0 +1,69 @@ +--- +name: user-management +description: Manage users, groups, and permissions on Linux systems. Configure sudo and access controls. Use when managing system access. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# User Management + +Manage users, groups, and permissions. + +## User Operations + +```bash +# Create user +useradd -m -s /bin/bash username +passwd username + +# Delete user +userdel -r username + +# Modify user +usermod -aG sudo username +usermod -s /bin/zsh username +``` + +## Group Management + +```bash +# Create group +groupadd developers + +# Add user to group +usermod -aG developers username +gpasswd -a username developers + +# Remove from group +gpasswd -d username developers +``` + +## Sudo Configuration + +```bash +# /etc/sudoers.d/developers +%developers ALL=(ALL) NOPASSWD: /usr/bin/docker +username ALL=(ALL) NOPASSWD: ALL +``` + +## File Permissions + +```bash +chmod 755 file # rwxr-xr-x +chmod u+x file # Add execute for user +chown user:group file # Change ownership +chown -R user:group dir/ + +# ACLs +setfacl -m u:user:rx file +getfacl file +``` + +## Best Practices + +- Use groups for access control +- Minimal sudo privileges +- Regular access reviews +- Strong password policies diff --git a/infrastructure/servers/windows-server/SKILL.md b/infrastructure/servers/windows-server/SKILL.md new file mode 100644 index 0000000..b2b1c28 --- /dev/null +++ b/infrastructure/servers/windows-server/SKILL.md @@ -0,0 +1,56 @@ +--- +name: windows-server +description: Administer Windows Server systems. Manage IIS, Active Directory, and PowerShell automation. Use when administering Windows infrastructure. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# Windows Server Administration + +Windows Server management and PowerShell automation. + +## Server Roles + +```powershell +# Install IIS +Install-WindowsFeature -Name Web-Server -IncludeManagementTools + +# Install AD DS +Install-WindowsFeature -Name AD-Domain-Services -IncludeManagementTools + +# List installed features +Get-WindowsFeature | Where-Object Installed +``` + +## System Information + +```powershell +Get-ComputerInfo +Get-Process +Get-Service +Get-EventLog -LogName System -Newest 50 +``` + +## IIS Management + +```powershell +# Create website +New-Website -Name "MyApp" -Port 80 -PhysicalPath "C:\inetpub\myapp" + +# Create app pool +New-WebAppPool -Name "MyAppPool" + +# Start/Stop +Start-Website -Name "MyApp" +Stop-Website -Name "MyApp" +``` + +## Best Practices + +- Use Server Core when possible +- Implement Windows Admin Center +- Regular Windows Update +- PowerShell remoting over WinRM +- Active Directory best practices diff --git a/infrastructure/storage/backup-recovery/SKILL.md b/infrastructure/storage/backup-recovery/SKILL.md new file mode 100644 index 0000000..8270c2b --- /dev/null +++ b/infrastructure/storage/backup-recovery/SKILL.md @@ -0,0 +1,63 @@ +--- +name: backup-recovery +description: Implement backup and recovery strategies. Configure rsync, Restic, and cloud backups. Use when designing data protection solutions. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# Backup and Recovery + +Implement comprehensive backup strategies. + +## rsync Backups + +```bash +# Basic sync +rsync -avz --delete /source/ /backup/ + +# Remote backup +rsync -avz -e ssh /data/ user@backup:/backups/ + +# Incremental with hard links +rsync -avz --delete --link-dest=/backup/latest /source/ /backup/$(date +%Y%m%d)/ +``` + +## Restic Backup + +```bash +# Initialize repository +restic init --repo /backups + +# Backup +restic backup /data --repo /backups + +# List snapshots +restic snapshots --repo /backups + +# Restore +restic restore latest --target /restore --repo /backups + +# Prune old backups +restic forget --keep-daily 7 --keep-weekly 4 --keep-monthly 12 --prune +``` + +## Cloud Backup + +```bash +# AWS S3 with restic +restic init --repo s3:s3.amazonaws.com/bucket-name +restic backup /data --repo s3:s3.amazonaws.com/bucket-name + +# GCS +restic init --repo gs:bucket-name:/ +``` + +## Best Practices + +- Follow 3-2-1 rule +- Test recovery regularly +- Encrypt backups +- Document procedures +- Monitor backup success diff --git a/infrastructure/storage/block-storage/SKILL.md b/infrastructure/storage/block-storage/SKILL.md new file mode 100644 index 0000000..c3a81eb --- /dev/null +++ b/infrastructure/storage/block-storage/SKILL.md @@ -0,0 +1,56 @@ +--- +name: block-storage +description: Manage block storage volumes and LVM. Configure cloud block storage and local disks. Use when managing disk storage. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# Block Storage + +Manage block storage volumes and LVM. + +## LVM Management + +```bash +# Create physical volume +pvcreate /dev/sdb + +# Create volume group +vgcreate data_vg /dev/sdb + +# Create logical volume +lvcreate -L 50G -n app_lv data_vg + +# Format and mount +mkfs.ext4 /dev/data_vg/app_lv +mount /dev/data_vg/app_lv /data + +# Extend volume +lvextend -L +10G /dev/data_vg/app_lv +resize2fs /dev/data_vg/app_lv +``` + +## AWS EBS + +```bash +# Create volume +aws ec2 create-volume \ + --availability-zone us-east-1a \ + --size 100 \ + --volume-type gp3 + +# Attach to instance +aws ec2 attach-volume \ + --volume-id vol-xxx \ + --instance-id i-xxx \ + --device /dev/xvdf +``` + +## Best Practices + +- Use LVM for flexibility +- Implement RAID for redundancy +- Monitor disk I/O +- Regular disk health checks diff --git a/infrastructure/storage/nfs-storage/SKILL.md b/infrastructure/storage/nfs-storage/SKILL.md new file mode 100644 index 0000000..0adcf9b --- /dev/null +++ b/infrastructure/storage/nfs-storage/SKILL.md @@ -0,0 +1,67 @@ +--- +name: nfs-storage +description: Configure NFS servers and clients. Implement network file sharing for Linux systems. Use when setting up shared storage. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# NFS Storage + +Configure NFS for network file sharing. + +## Server Configuration + +```bash +# Install +apt install nfs-kernel-server + +# Configure exports +# /etc/exports +/data 10.0.0.0/24(rw,sync,no_subtree_check,no_root_squash) +/shared *(ro,sync,no_subtree_check) + +# Apply changes +exportfs -ra + +# Start service +systemctl enable --now nfs-kernel-server +``` + +## Client Configuration + +```bash +# Install +apt install nfs-common + +# Mount +mount -t nfs server:/data /mnt/data + +# /etc/fstab +server:/data /mnt/data nfs defaults,_netdev 0 0 +``` + +## Kubernetes NFS + +```yaml +apiVersion: v1 +kind: PersistentVolume +metadata: + name: nfs-pv +spec: + capacity: + storage: 100Gi + accessModes: + - ReadWriteMany + nfs: + server: nfs-server.example.com + path: /data +``` + +## Best Practices + +- Use proper export options +- Implement firewall rules +- Monitor NFS performance +- Use NFSv4 for security diff --git a/infrastructure/storage/object-storage/SKILL.md b/infrastructure/storage/object-storage/SKILL.md new file mode 100644 index 0000000..615e51e --- /dev/null +++ b/infrastructure/storage/object-storage/SKILL.md @@ -0,0 +1,52 @@ +--- +name: object-storage +description: Configure object storage with S3, GCS, and MinIO. Implement lifecycle policies and access controls. Use when managing object storage. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# Object Storage + +Configure and manage object storage solutions. + +## AWS S3 + +```bash +# Create bucket +aws s3 mb s3://my-bucket + +# Upload/Download +aws s3 cp file.txt s3://my-bucket/ +aws s3 sync ./local s3://my-bucket/remote + +# Configure lifecycle +aws s3api put-bucket-lifecycle-configuration \ + --bucket my-bucket \ + --lifecycle-configuration file://lifecycle.json +``` + +## MinIO (Self-Hosted) + +```bash +# Deploy +docker run -d \ + -p 9000:9000 -p 9001:9001 \ + -e MINIO_ROOT_USER=admin \ + -e MINIO_ROOT_PASSWORD=password \ + -v /data:/data \ + minio/minio server /data --console-address ":9001" + +# Configure mc client +mc alias set myminio http://localhost:9000 admin password +mc mb myminio/mybucket +``` + +## Best Practices + +- Enable versioning +- Implement lifecycle policies +- Use server-side encryption +- Configure access logging +- Implement bucket policies diff --git a/security/hardening/cis-benchmarks/SKILL.md b/security/hardening/cis-benchmarks/SKILL.md new file mode 100644 index 0000000..c542a7c --- /dev/null +++ b/security/hardening/cis-benchmarks/SKILL.md @@ -0,0 +1,126 @@ +--- +name: cis-benchmarks +description: Audit and remediate CIS benchmark violations. Use automated tools to assess compliance and implement hardening recommendations. Use when meeting compliance requirements or implementing security baselines. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# CIS Benchmarks + +Implement and audit CIS security benchmarks. + +## When to Use This Skill + +Use this skill when: +- Assessing security compliance +- Implementing security baselines +- Meeting regulatory requirements +- Hardening systems to standards + +## Assessment Tools + +### OpenSCAP + +```bash +# Install +apt install openscap-scanner scap-security-guide + +# Run CIS benchmark scan +oscap xccdf eval \ + --profile xccdf_org.ssgproject.content_profile_cis \ + --results results.xml \ + --report report.html \ + /usr/share/xml/scap/ssg/content/ssg-ubuntu2204-ds.xml +``` + +### Lynis + +```bash +# Install +apt install lynis + +# Run audit +lynis audit system + +# Generate report +lynis audit system --report-file /tmp/lynis-report.dat +``` + +### InSpec + +```ruby +# cis-profile/controls/ssh.rb +control 'cis-ssh-1' do + impact 1.0 + title 'Ensure SSH root login is disabled' + + describe sshd_config do + its('PermitRootLogin') { should eq 'no' } + end +end + +control 'cis-ssh-2' do + impact 0.7 + title 'Ensure SSH password authentication is disabled' + + describe sshd_config do + its('PasswordAuthentication') { should eq 'no' } + end +end +``` + +```bash +# Run InSpec +inspec exec cis-profile -t ssh://user@target +``` + +### Kubernetes CIS + +```bash +# kube-bench +docker run --rm -v /etc:/etc:ro -v /var:/var:ro \ + aquasec/kube-bench:latest run --targets node + +# Check specific sections +kube-bench run --targets master --check 1.1,1.2 +``` + +## Remediation Workflow + +```yaml +workflow: + 1_scan: + - Run automated assessment + - Generate baseline report + + 2_analyze: + - Review findings + - Identify false positives + - Prioritize by risk + + 3_remediate: + - Apply fixes + - Document exceptions + - Verify changes + + 4_validate: + - Re-run assessment + - Confirm remediation + - Generate compliance report +``` + +## Best Practices + +- Baseline before hardening +- Document exceptions +- Automate assessments +- Track compliance over time +- Regular re-assessment +- Version control configurations + +## Related Skills + +- [linux-hardening](../linux-hardening/) - Linux security +- [vulnerability-scanning](../../scanning/vulnerability-scanning/) - Security scanning diff --git a/security/hardening/container-hardening/SKILL.md b/security/hardening/container-hardening/SKILL.md new file mode 100644 index 0000000..21c3f37 --- /dev/null +++ b/security/hardening/container-hardening/SKILL.md @@ -0,0 +1,102 @@ +--- +name: container-hardening +description: Secure Docker images and container runtime configurations. Implement non-root users, read-only filesystems, and security contexts. Use when building secure container images or hardening container deployments. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# Container Hardening + +Secure container images and runtime configurations. + +## When to Use This Skill + +Use this skill when: +- Building secure container images +- Hardening container deployments +- Meeting container security requirements +- Implementing defense in depth + +## Dockerfile Security + +```dockerfile +# Use minimal base image +FROM alpine:3.18 + +# Don't run as root +RUN addgroup -g 1001 -S appgroup && \ + adduser -u 1001 -S appuser -G appgroup + +# Copy with specific ownership +COPY --chown=appuser:appgroup . /app + +# Remove unnecessary packages +RUN apk del --purge build-dependencies && \ + rm -rf /var/cache/apk/* + +# Use non-root user +USER appuser + +# Read-only filesystem support +WORKDIR /app +``` + +## Runtime Security + +```bash +# Run with security options +docker run -d \ + --read-only \ + --tmpfs /tmp \ + --security-opt=no-new-privileges:true \ + --cap-drop=ALL \ + --cap-add=NET_BIND_SERVICE \ + --user 1001:1001 \ + myapp:latest +``` + +## Kubernetes Security Context + +```yaml +apiVersion: v1 +kind: Pod +spec: + securityContext: + runAsNonRoot: true + runAsUser: 1001 + fsGroup: 1001 + containers: + - name: app + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] +``` + +## Image Scanning + +```bash +# Scan with Trivy +trivy image --severity HIGH,CRITICAL myapp:latest + +# Use distroless images +FROM gcr.io/distroless/static-debian11 +``` + +## Best Practices + +- Use minimal base images +- Run as non-root user +- Enable read-only filesystem +- Drop all capabilities +- Scan images regularly +- Sign and verify images +- Use secrets management + +## Related Skills + +- [container-scanning](../../scanning/container-scanning/) - Vulnerability scanning +- [kubernetes-hardening](../kubernetes-hardening/) - K8s security diff --git a/security/hardening/container-hardening/references/container-security.md b/security/hardening/container-hardening/references/container-security.md new file mode 100644 index 0000000..eb14872 --- /dev/null +++ b/security/hardening/container-hardening/references/container-security.md @@ -0,0 +1,105 @@ +# Container Security Best Practices + +## Dockerfile Hardening + +```dockerfile +# Use minimal base image +FROM gcr.io/distroless/base-debian12 + +# Or Alpine +FROM alpine:3.19 + +# Non-root user +RUN addgroup -g 1000 appgroup && \ + adduser -u 1000 -G appgroup -D appuser +USER appuser + +# Read-only filesystem +# (Set at runtime with --read-only) + +# No new privileges +# (Set at runtime with --security-opt=no-new-privileges) +``` + +## Security Scanning + +```bash +# Trivy scan +trivy image --severity HIGH,CRITICAL myimage:latest + +# Grype scan +grype myimage:latest --fail-on high + +# Docker Scout +docker scout cves myimage:latest +``` + +## Runtime Security + +```yaml +# Kubernetes securityContext +securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault +``` + +## Docker Run Hardening + +```bash +docker run \ + --read-only \ + --tmpfs /tmp \ + --security-opt=no-new-privileges:true \ + --cap-drop=ALL \ + --user 1000:1000 \ + --memory=512m \ + --cpus=0.5 \ + myimage +``` + +## Image Signing + +```bash +# Cosign +cosign sign --key cosign.key myimage:latest +cosign verify --key cosign.pub myimage:latest + +# Docker Content Trust +export DOCKER_CONTENT_TRUST=1 +docker push myimage:latest +``` + +## Network Policies + +```yaml +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: deny-all +spec: + podSelector: {} + policyTypes: + - Ingress + - Egress +``` + +## Checklist + +- [ ] Use minimal base images +- [ ] Run as non-root +- [ ] Drop all capabilities +- [ ] Read-only filesystem +- [ ] No privilege escalation +- [ ] Scan for vulnerabilities +- [ ] Sign images +- [ ] Implement network policies +- [ ] Use secrets management +- [ ] Enable audit logging diff --git a/security/hardening/kubernetes-hardening/SKILL.md b/security/hardening/kubernetes-hardening/SKILL.md new file mode 100644 index 0000000..51ed4c8 --- /dev/null +++ b/security/hardening/kubernetes-hardening/SKILL.md @@ -0,0 +1,129 @@ +--- +name: kubernetes-hardening +description: Implement Kubernetes security contexts, Pod Security Standards, and network policies. Secure cluster components and workloads. Use when hardening Kubernetes deployments or meeting security compliance. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# Kubernetes Hardening + +Secure Kubernetes clusters and workloads. + +## When to Use This Skill + +Use this skill when: +- Hardening Kubernetes clusters +- Implementing Pod Security Standards +- Configuring network policies +- Meeting security compliance + +## Pod Security Standards + +```yaml +# Namespace with restricted policy +apiVersion: v1 +kind: Namespace +metadata: + name: production + labels: + pod-security.kubernetes.io/enforce: restricted + pod-security.kubernetes.io/audit: restricted + pod-security.kubernetes.io/warn: restricted +``` + +## Security Context + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: secure-pod +spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + fsGroup: 1000 + seccompProfile: + type: RuntimeDefault + containers: + - name: app + image: myapp:latest + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] +``` + +## Network Policies + +```yaml +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: default-deny-all +spec: + podSelector: {} + policyTypes: + - Ingress + - Egress +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-web +spec: + podSelector: + matchLabels: + app: web + ingress: + - from: + - podSelector: + matchLabels: + app: frontend + ports: + - port: 8080 +``` + +## RBAC + +```yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: app-reader +rules: +- apiGroups: [""] + resources: ["pods", "services"] + verbs: ["get", "list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: app-reader-binding +subjects: +- kind: ServiceAccount + name: myapp +roleRef: + kind: Role + name: app-reader + apiGroup: rbac.authorization.k8s.io +``` + +## Best Practices + +- Enable Pod Security Standards +- Implement network policies +- Use RBAC with least privilege +- Enable audit logging +- Secure etcd with encryption +- Use service mesh for mTLS +- Regular security scanning + +## Related Skills + +- [kubernetes-ops](../../../devops/orchestration/kubernetes-ops/) - K8s operations +- [container-hardening](../container-hardening/) - Container security diff --git a/security/hardening/linux-hardening/SKILL.md b/security/hardening/linux-hardening/SKILL.md new file mode 100644 index 0000000..8751949 --- /dev/null +++ b/security/hardening/linux-hardening/SKILL.md @@ -0,0 +1,130 @@ +--- +name: linux-hardening +description: Apply CIS benchmarks and secure Linux servers. Configure SSH, manage users, implement firewall rules, and enable security features. Use when hardening Linux systems for production or meeting security compliance requirements. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# Linux Hardening + +Secure Linux servers following CIS benchmarks and security best practices. + +## When to Use This Skill + +Use this skill when: +- Hardening production servers +- Meeting compliance requirements +- Implementing security baselines +- Configuring secure SSH access + +## SSH Hardening + +```bash +# /etc/ssh/sshd_config +PermitRootLogin no +PasswordAuthentication no +PubkeyAuthentication yes +MaxAuthTries 3 +ClientAliveInterval 300 +ClientAliveCountMax 2 +AllowUsers deploy admin +Protocol 2 +``` + +## User Security + +```bash +# Password policy +sudo apt install libpam-pwquality +# /etc/security/pwquality.conf +minlen = 14 +dcredit = -1 +ucredit = -1 +ocredit = -1 +lcredit = -1 + +# Lock inactive accounts +useradd -D -f 30 + +# Audit sudo usage +echo "Defaults logfile=/var/log/sudo.log" >> /etc/sudoers +``` + +## Firewall Configuration + +```bash +# UFW setup +ufw default deny incoming +ufw default allow outgoing +ufw allow ssh +ufw allow 443/tcp +ufw enable + +# Or iptables +iptables -P INPUT DROP +iptables -P FORWARD DROP +iptables -A INPUT -i lo -j ACCEPT +iptables -A INPUT -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT +iptables -A INPUT -p tcp --dport 22 -j ACCEPT +``` + +## Kernel Hardening + +```bash +# /etc/sysctl.d/99-security.conf +net.ipv4.conf.all.send_redirects = 0 +net.ipv4.conf.all.accept_redirects = 0 +net.ipv4.conf.all.accept_source_route = 0 +net.ipv4.icmp_echo_ignore_broadcasts = 1 +kernel.randomize_va_space = 2 +fs.suid_dumpable = 0 + +# Apply +sysctl -p +``` + +## File Permissions + +```bash +# Critical files +chmod 600 /etc/shadow +chmod 644 /etc/passwd +chmod 700 /root +chmod 600 /etc/ssh/sshd_config + +# Find world-writable files +find / -type f -perm -0002 -ls + +# Find SUID files +find / -perm -4000 -type f -ls +``` + +## Audit Configuration + +```bash +# Install auditd +apt install auditd + +# /etc/audit/rules.d/audit.rules +-w /etc/passwd -p wa -k identity +-w /etc/shadow -p wa -k identity +-w /etc/sudoers -p wa -k actions +-a always,exit -F arch=b64 -S execve -k exec +``` + +## Best Practices + +- Disable unused services +- Keep system updated +- Use fail2ban for intrusion prevention +- Enable SELinux/AppArmor +- Regular security audits +- Monitor log files +- Implement least privilege + +## Related Skills + +- [cis-benchmarks](../cis-benchmarks/) - Compliance scanning +- [firewall-config](../../network/firewall-config/) - Firewall rules diff --git a/security/hardening/linux-hardening/assets/ssh-hardening.conf b/security/hardening/linux-hardening/assets/ssh-hardening.conf new file mode 100644 index 0000000..c880370 --- /dev/null +++ b/security/hardening/linux-hardening/assets/ssh-hardening.conf @@ -0,0 +1,111 @@ +# SSH Server Hardening Configuration +# Place in /etc/ssh/sshd_config.d/hardening.conf +# Restart SSH: systemctl restart sshd + +#------------------------------------------------------------------------------ +# AUTHENTICATION +#------------------------------------------------------------------------------ + +# Disable root login +PermitRootLogin no + +# Disable password authentication +PasswordAuthentication no + +# Enable public key authentication +PubkeyAuthentication yes + +# Disable empty passwords +PermitEmptyPasswords no + +# Disable keyboard-interactive authentication +KbdInteractiveAuthentication no + +# Disable challenge-response authentication +ChallengeResponseAuthentication no + +# Maximum authentication attempts +MaxAuthTries 3 + +# Maximum sessions per connection +MaxSessions 2 + +# Maximum simultaneous unauthenticated connections +MaxStartups 10:30:60 + +# Login grace time +LoginGraceTime 60 + +#------------------------------------------------------------------------------ +# SESSION +#------------------------------------------------------------------------------ + +# Client alive settings (timeout) +ClientAliveInterval 300 +ClientAliveCountMax 2 + +# Disable TCP forwarding +AllowTcpForwarding no + +# Disable agent forwarding +AllowAgentForwarding no + +# Disable stream local forwarding +AllowStreamLocalForwarding no + +# Disable X11 forwarding +X11Forwarding no + +# Disable user environment processing +PermitUserEnvironment no + +# Disable tunnel device forwarding +PermitTunnel no + +# Disable gateway ports +GatewayPorts no + +#------------------------------------------------------------------------------ +# CRYPTOGRAPHY +#------------------------------------------------------------------------------ + +# Protocol version (SSH-2 only) +Protocol 2 + +# Strong ciphers only +Ciphers aes256-gcm@openssh.com,chacha20-poly1305@openssh.com,aes256-ctr + +# Strong MACs only +MACs hmac-sha2-512-etm@openssh.com,hmac-sha2-256-etm@openssh.com,hmac-sha2-512,hmac-sha2-256 + +# Strong key exchange algorithms +KexAlgorithms curve25519-sha256,curve25519-sha256@libssh.org,diffie-hellman-group16-sha512,diffie-hellman-group18-sha512 + +# Strong host key algorithms +HostKeyAlgorithms ssh-ed25519,rsa-sha2-512,rsa-sha2-256 + +#------------------------------------------------------------------------------ +# LOGGING +#------------------------------------------------------------------------------ + +# Log level +LogLevel VERBOSE + +# Enable sftp logging +Subsystem sftp /usr/lib/openssh/sftp-server -l INFO + +#------------------------------------------------------------------------------ +# ACCESS CONTROL +#------------------------------------------------------------------------------ + +# Use PAM +UsePAM yes + +# Show banner +Banner /etc/issue.net + +# Restrict to specific users (uncomment and customize) +# AllowUsers admin deploy + +# Restrict to specific groups (uncomment and customize) +# AllowGroups sshusers admins diff --git a/security/hardening/linux-hardening/assets/sysctl-hardening.conf b/security/hardening/linux-hardening/assets/sysctl-hardening.conf new file mode 100644 index 0000000..3839fd2 --- /dev/null +++ b/security/hardening/linux-hardening/assets/sysctl-hardening.conf @@ -0,0 +1,104 @@ +# Linux Kernel Security Hardening +# Place in /etc/sysctl.d/99-security.conf +# Apply with: sysctl -p /etc/sysctl.d/99-security.conf + +#------------------------------------------------------------------------------ +# NETWORK SECURITY +#------------------------------------------------------------------------------ + +# Disable IP forwarding (unless router) +net.ipv4.ip_forward = 0 +net.ipv6.conf.all.forwarding = 0 + +# Disable packet redirect sending +net.ipv4.conf.all.send_redirects = 0 +net.ipv4.conf.default.send_redirects = 0 + +# Disable ICMP redirect acceptance +net.ipv4.conf.all.accept_redirects = 0 +net.ipv4.conf.default.accept_redirects = 0 +net.ipv4.conf.all.secure_redirects = 0 +net.ipv4.conf.default.secure_redirects = 0 +net.ipv6.conf.all.accept_redirects = 0 +net.ipv6.conf.default.accept_redirects = 0 + +# Disable source routing +net.ipv4.conf.all.accept_source_route = 0 +net.ipv4.conf.default.accept_source_route = 0 +net.ipv6.conf.all.accept_source_route = 0 +net.ipv6.conf.default.accept_source_route = 0 + +# Log suspicious packets +net.ipv4.conf.all.log_martians = 1 +net.ipv4.conf.default.log_martians = 1 + +# Ignore ICMP broadcast requests +net.ipv4.icmp_echo_ignore_broadcasts = 1 + +# Ignore bogus ICMP error responses +net.ipv4.icmp_ignore_bogus_error_responses = 1 + +# Enable reverse path filtering (spoofing protection) +net.ipv4.conf.all.rp_filter = 1 +net.ipv4.conf.default.rp_filter = 1 + +# Enable TCP SYN cookies (SYN flood protection) +net.ipv4.tcp_syncookies = 1 + +# Disable IPv6 router advertisements +net.ipv6.conf.all.accept_ra = 0 +net.ipv6.conf.default.accept_ra = 0 + +# Disable IPv6 if not needed +# net.ipv6.conf.all.disable_ipv6 = 1 +# net.ipv6.conf.default.disable_ipv6 = 1 + +#------------------------------------------------------------------------------ +# KERNEL SECURITY +#------------------------------------------------------------------------------ + +# Enable ASLR +kernel.randomize_va_space = 2 + +# Restrict access to kernel pointers +kernel.kptr_restrict = 2 + +# Restrict dmesg access +kernel.dmesg_restrict = 1 + +# Restrict ptrace scope +kernel.yama.ptrace_scope = 1 + +# Disable magic SysRq key +kernel.sysrq = 0 + +# Restrict unprivileged user namespaces +# kernel.unprivileged_userns_clone = 0 + +# Restrict loading TTY line disciplines +dev.tty.ldisc_autoload = 0 + +# Restrict userfaultfd to privileged users +vm.unprivileged_userfaultfd = 0 + +# Restrict BPF +kernel.unprivileged_bpf_disabled = 1 +net.core.bpf_jit_harden = 2 + +# Restrict perf events +kernel.perf_event_paranoid = 3 + +#------------------------------------------------------------------------------ +# FILE SYSTEM SECURITY +#------------------------------------------------------------------------------ + +# Restrict core dumps +fs.suid_dumpable = 0 + +# Restrict hardlinks and symlinks +fs.protected_hardlinks = 1 +fs.protected_symlinks = 1 + +# Protect FIFOs and regular files +fs.protected_fifos = 2 +fs.protected_regular = 2 diff --git a/security/hardening/linux-hardening/references/cis-checklist.md b/security/hardening/linux-hardening/references/cis-checklist.md new file mode 100644 index 0000000..4b6f319 --- /dev/null +++ b/security/hardening/linux-hardening/references/cis-checklist.md @@ -0,0 +1,139 @@ +# CIS Linux Hardening Checklist + +## 1. Initial Setup + +### 1.1 Filesystem Configuration +- [ ] Disable unused filesystems (cramfs, freevxfs, jffs2, hfs, hfsplus, squashfs, udf) +- [ ] Ensure `/tmp` is configured with nodev, nosuid, noexec +- [ ] Ensure `/var`, `/var/tmp`, `/var/log`, `/var/log/audit` are separate partitions +- [ ] Ensure `/home` is separate partition with nodev + +### 1.2 Configure Software Updates +- [ ] Ensure package manager repositories are configured +- [ ] Ensure GPG keys are configured +- [ ] Ensure automatic updates are enabled + +### 1.3 Filesystem Integrity +- [ ] Ensure AIDE is installed +- [ ] Ensure filesystem integrity is regularly checked + +## 2. Services + +### 2.1 Special Purpose Services +- [ ] Ensure time synchronization is configured (chrony/ntp) +- [ ] Ensure X Window System is not installed +- [ ] Ensure rsync service is not installed or masked +- [ ] Ensure Avahi Server is not installed +- [ ] Ensure CUPS is not installed +- [ ] Ensure DHCP Server is not installed +- [ ] Ensure LDAP server is not installed +- [ ] Ensure NFS is not installed +- [ ] Ensure DNS Server is not installed +- [ ] Ensure FTP Server is not installed +- [ ] Ensure HTTP Server is not installed +- [ ] Ensure IMAP and POP3 server is not installed +- [ ] Ensure Samba is not installed +- [ ] Ensure SNMP Server is not installed + +### 2.2 Service Clients +- [ ] Ensure NIS Client is not installed +- [ ] Ensure rsh client is not installed +- [ ] Ensure talk client is not installed +- [ ] Ensure telnet client is not installed +- [ ] Ensure LDAP client is not installed +- [ ] Ensure RPC is not installed + +## 3. Network Configuration + +### 3.1 Network Parameters (Host Only) +- [ ] Ensure IP forwarding is disabled +- [ ] Ensure packet redirect sending is disabled + +### 3.2 Network Parameters (Host and Router) +- [ ] Ensure source routed packets are not accepted +- [ ] Ensure ICMP redirects are not accepted +- [ ] Ensure secure ICMP redirects are not accepted +- [ ] Ensure suspicious packets are logged +- [ ] Ensure broadcast ICMP requests are ignored +- [ ] Ensure bogus ICMP responses are ignored +- [ ] Ensure Reverse Path Filtering is enabled +- [ ] Ensure TCP SYN Cookies is enabled + +### 3.3 Firewall Configuration +- [ ] Ensure firewall is installed (iptables, nftables, or firewalld) +- [ ] Ensure default deny firewall policy +- [ ] Ensure loopback traffic is configured +- [ ] Ensure outbound connections are configured + +## 4. Access, Authentication and Authorization + +### 4.1 Configure Shadow Suite +- [ ] Ensure password expiration is 365 days or less +- [ ] Ensure minimum days between password changes is 7 or more +- [ ] Ensure password expiration warning days is 7 or more +- [ ] Ensure inactive password lock is 30 days or less +- [ ] Ensure all users last password change date is in the past + +### 4.2 Configure SSH Server +- [ ] Ensure SSH Protocol is set to 2 +- [ ] Ensure SSH LogLevel is appropriate +- [ ] Ensure SSH X11 forwarding is disabled +- [ ] Ensure SSH MaxAuthTries is set to 4 or less +- [ ] Ensure SSH IgnoreRhosts is enabled +- [ ] Ensure SSH HostbasedAuthentication is disabled +- [ ] Ensure SSH root login is disabled +- [ ] Ensure SSH PermitEmptyPasswords is disabled +- [ ] Ensure SSH PermitUserEnvironment is disabled +- [ ] Ensure SSH Idle Timeout Interval is configured +- [ ] Ensure SSH LoginGraceTime is set to one minute or less +- [ ] Ensure SSH warning banner is configured +- [ ] Ensure SSH PAM is enabled +- [ ] Ensure SSH AllowTcpForwarding is disabled + +### 4.3 Configure PAM +- [ ] Ensure password creation requirements are configured +- [ ] Ensure lockout for failed password attempts is configured +- [ ] Ensure password reuse is limited +- [ ] Ensure password hashing algorithm is SHA-512 + +## 5. Logging and Auditing + +### 5.1 Configure Logging +- [ ] Ensure rsyslog is installed +- [ ] Ensure rsyslog Service is enabled +- [ ] Ensure logging is configured +- [ ] Ensure rsyslog default file permissions configured +- [ ] Ensure remote rsyslog messages only accepted on designated log hosts + +### 5.2 Configure auditd +- [ ] Ensure auditing is enabled +- [ ] Ensure audit log storage size is configured +- [ ] Ensure audit logs are not automatically deleted +- [ ] Ensure changes to system administration scope are collected +- [ ] Ensure login and logout events are collected +- [ ] Ensure session initiation information is collected +- [ ] Ensure file deletion events by users are collected +- [ ] Ensure kernel module loading and unloading is collected + +## 6. System Maintenance + +### 6.1 File Permissions +- [ ] Ensure permissions on /etc/passwd are configured (644) +- [ ] Ensure permissions on /etc/shadow are configured (600) +- [ ] Ensure permissions on /etc/group are configured (644) +- [ ] Ensure permissions on /etc/gshadow are configured (600) +- [ ] Ensure no world writable files exist +- [ ] Ensure no unowned files or directories exist +- [ ] Ensure no ungrouped files or directories exist + +### 6.2 User and Group Settings +- [ ] Ensure accounts in /etc/passwd use shadowed passwords +- [ ] Ensure no legacy "+" entries exist in /etc/passwd +- [ ] Ensure root is the only UID 0 account +- [ ] Ensure root PATH integrity +- [ ] Ensure all users' home directories exist +- [ ] Ensure users' home directories permissions are 750 or more restrictive +- [ ] Ensure users own their home directories +- [ ] Ensure no users have .forward files +- [ ] Ensure no users have .netrc files +- [ ] Ensure no users have .rhosts files diff --git a/security/hardening/linux-hardening/scripts/audit-system.sh b/security/hardening/linux-hardening/scripts/audit-system.sh new file mode 100644 index 0000000..f202ac9 --- /dev/null +++ b/security/hardening/linux-hardening/scripts/audit-system.sh @@ -0,0 +1,138 @@ +#!/bin/bash +# Linux Security Audit Script +# Usage: ./audit-system.sh [--verbose] + +set -euo pipefail + +VERBOSE="${1:-}" +PASS=0 +WARN=0 +FAIL=0 + +check() { + local status="$1" + local message="$2" + case "$status" in + PASS) echo -e "\e[32m[PASS]\e[0m $message"; ((PASS++)) ;; + WARN) echo -e "\e[33m[WARN]\e[0m $message"; ((WARN++)) ;; + FAIL) echo -e "\e[31m[FAIL]\e[0m $message"; ((FAIL++)) ;; + esac +} + +echo "=========================================" +echo "Linux Security Audit" +echo "=========================================" +echo "" + +# 1. System Updates +echo "1. System Updates" +echo "-----------------" +UPDATES=$(apt-get -s upgrade 2>/dev/null | grep -c "^Inst" || echo 0) +if [ "$UPDATES" -eq 0 ]; then + check "PASS" "System is up to date" +else + check "FAIL" "$UPDATES packages need updating" +fi + +# 2. SSH Configuration +echo "" +echo "2. SSH Configuration" +echo "--------------------" +if grep -q "^PermitRootLogin no" /etc/ssh/sshd_config* 2>/dev/null; then + check "PASS" "Root login disabled" +else + check "FAIL" "Root login may be enabled" +fi + +if grep -q "^PasswordAuthentication no" /etc/ssh/sshd_config* 2>/dev/null; then + check "PASS" "Password authentication disabled" +else + check "WARN" "Password authentication may be enabled" +fi + +# 3. User Accounts +echo "" +echo "3. User Accounts" +echo "----------------" +EMPTY_PASS=$(awk -F: '($2 == "") {print $1}' /etc/shadow 2>/dev/null | wc -l) +if [ "$EMPTY_PASS" -eq 0 ]; then + check "PASS" "No accounts with empty passwords" +else + check "FAIL" "$EMPTY_PASS accounts with empty passwords" +fi + +ROOT_ACCOUNTS=$(awk -F: '($3 == 0) {print $1}' /etc/passwd | wc -l) +if [ "$ROOT_ACCOUNTS" -eq 1 ]; then + check "PASS" "Only root has UID 0" +else + check "FAIL" "$ROOT_ACCOUNTS accounts have UID 0" +fi + +# 4. File Permissions +echo "" +echo "4. File Permissions" +echo "-------------------" +SHADOW_PERMS=$(stat -c %a /etc/shadow 2>/dev/null) +if [ "$SHADOW_PERMS" = "600" ] || [ "$SHADOW_PERMS" = "640" ]; then + check "PASS" "/etc/shadow permissions: $SHADOW_PERMS" +else + check "FAIL" "/etc/shadow permissions: $SHADOW_PERMS (should be 600)" +fi + +WORLD_WRITABLE=$(find /etc -type f -perm -002 2>/dev/null | wc -l) +if [ "$WORLD_WRITABLE" -eq 0 ]; then + check "PASS" "No world-writable files in /etc" +else + check "FAIL" "$WORLD_WRITABLE world-writable files in /etc" +fi + +# 5. Network Security +echo "" +echo "5. Network Security" +echo "-------------------" +if sysctl -n net.ipv4.tcp_syncookies 2>/dev/null | grep -q "1"; then + check "PASS" "TCP SYN cookies enabled" +else + check "WARN" "TCP SYN cookies not enabled" +fi + +if sysctl -n net.ipv4.conf.all.rp_filter 2>/dev/null | grep -q "1"; then + check "PASS" "Reverse path filtering enabled" +else + check "WARN" "Reverse path filtering not enabled" +fi + +# 6. Firewall +echo "" +echo "6. Firewall Status" +echo "------------------" +if command -v ufw &>/dev/null && ufw status | grep -q "active"; then + check "PASS" "UFW firewall is active" +elif command -v firewalld &>/dev/null && systemctl is-active firewalld &>/dev/null; then + check "PASS" "firewalld is active" +elif iptables -L -n 2>/dev/null | grep -q "DROP\|REJECT"; then + check "PASS" "iptables has rules configured" +else + check "FAIL" "No firewall appears to be active" +fi + +# 7. Services +echo "" +echo "7. Running Services" +echo "-------------------" +LISTENING=$(ss -tlnp 2>/dev/null | grep -c LISTEN || echo 0) +check "WARN" "$LISTENING services listening on ports" + +# Summary +echo "" +echo "=========================================" +echo "Audit Summary" +echo "=========================================" +echo -e "Passed: \e[32m$PASS\e[0m" +echo -e "Warnings: \e[33m$WARN\e[0m" +echo -e "Failed: \e[31m$FAIL\e[0m" +echo "" + +if [ "$FAIL" -gt 0 ]; then + exit 1 +fi diff --git a/security/hardening/linux-hardening/scripts/harden-system.sh b/security/hardening/linux-hardening/scripts/harden-system.sh new file mode 100644 index 0000000..fbd79be --- /dev/null +++ b/security/hardening/linux-hardening/scripts/harden-system.sh @@ -0,0 +1,131 @@ +#!/bin/bash +# Linux System Hardening Script +# Usage: ./harden-system.sh [--apply] +# Run without --apply to see what changes would be made + +set -euo pipefail + +APPLY="${1:-}" + +if [ "$APPLY" != "--apply" ]; then + echo "DRY RUN MODE - No changes will be made" + echo "Run with --apply to make changes" + echo "" +fi + +apply_change() { + if [ "$APPLY" == "--apply" ]; then + eval "$1" + echo " [APPLIED] $2" + else + echo " [WOULD APPLY] $2" + fi +} + +echo "=========================================" +echo "Linux System Hardening" +echo "=========================================" +echo "" + +# 1. Update system +echo "1. System Updates" +echo "-----------------" +apply_change "apt-get update && apt-get upgrade -y" "Update all packages" + +# 2. Disable unused filesystems +echo "" +echo "2. Disable Unused Filesystems" +echo "------------------------------" +FILESYSTEMS="cramfs freevxfs jffs2 hfs hfsplus squashfs udf" +for fs in $FILESYSTEMS; do + apply_change "echo 'install $fs /bin/true' >> /etc/modprobe.d/disable-filesystems.conf" "Disable $fs" +done + +# 3. Kernel parameters +echo "" +echo "3. Kernel Hardening (sysctl)" +echo "----------------------------" +SYSCTL_CONF="/etc/sysctl.d/99-hardening.conf" +cat << 'EOF' > /tmp/sysctl-hardening.conf +# Network security +net.ipv4.conf.all.send_redirects = 0 +net.ipv4.conf.default.send_redirects = 0 +net.ipv4.conf.all.accept_redirects = 0 +net.ipv4.conf.default.accept_redirects = 0 +net.ipv4.conf.all.secure_redirects = 0 +net.ipv4.conf.default.secure_redirects = 0 +net.ipv4.conf.all.log_martians = 1 +net.ipv4.conf.default.log_martians = 1 +net.ipv4.icmp_echo_ignore_broadcasts = 1 +net.ipv4.icmp_ignore_bogus_error_responses = 1 +net.ipv4.conf.all.rp_filter = 1 +net.ipv4.conf.default.rp_filter = 1 +net.ipv4.tcp_syncookies = 1 + +# IPv6 (disable if not needed) +net.ipv6.conf.all.disable_ipv6 = 1 +net.ipv6.conf.default.disable_ipv6 = 1 + +# Kernel hardening +kernel.randomize_va_space = 2 +kernel.kptr_restrict = 2 +kernel.dmesg_restrict = 1 +kernel.yama.ptrace_scope = 1 +EOF +apply_change "cp /tmp/sysctl-hardening.conf $SYSCTL_CONF && sysctl -p $SYSCTL_CONF" "Apply kernel hardening parameters" + +# 4. SSH hardening +echo "" +echo "4. SSH Hardening" +echo "----------------" +SSH_CONF="/etc/ssh/sshd_config.d/hardening.conf" +cat << 'EOF' > /tmp/ssh-hardening.conf +PermitRootLogin no +PasswordAuthentication no +PubkeyAuthentication yes +MaxAuthTries 3 +ClientAliveInterval 300 +ClientAliveCountMax 2 +X11Forwarding no +AllowAgentForwarding no +PermitEmptyPasswords no +EOF +apply_change "cp /tmp/ssh-hardening.conf $SSH_CONF" "Apply SSH hardening" + +# 5. File permissions +echo "" +echo "5. File Permissions" +echo "-------------------" +apply_change "chmod 600 /etc/shadow" "Secure /etc/shadow" +apply_change "chmod 644 /etc/passwd" "Secure /etc/passwd" +apply_change "chmod 600 /etc/gshadow" "Secure /etc/gshadow" +apply_change "chmod 644 /etc/group" "Secure /etc/group" + +# 6. Remove unnecessary packages +echo "" +echo "6. Remove Unnecessary Services" +echo "------------------------------" +REMOVE_PKGS="telnet rsh-client rsh-redone-client" +for pkg in $REMOVE_PKGS; do + apply_change "apt-get remove -y $pkg 2>/dev/null || true" "Remove $pkg" +done + +# 7. Configure firewall +echo "" +echo "7. Enable Firewall" +echo "------------------" +apply_change "ufw default deny incoming && ufw default allow outgoing && ufw allow ssh && ufw --force enable" "Configure UFW firewall" + +# 8. Enable automatic updates +echo "" +echo "8. Automatic Security Updates" +echo "-----------------------------" +apply_change "apt-get install -y unattended-upgrades && dpkg-reconfigure -plow unattended-upgrades" "Enable unattended upgrades" + +echo "" +echo "=========================================" +echo "Hardening script complete" +if [ "$APPLY" != "--apply" ]; then + echo "Run with --apply to make changes" +fi +echo "=========================================" diff --git a/security/hardening/windows-hardening/SKILL.md b/security/hardening/windows-hardening/SKILL.md new file mode 100644 index 0000000..df99444 --- /dev/null +++ b/security/hardening/windows-hardening/SKILL.md @@ -0,0 +1,98 @@ +--- +name: windows-hardening +description: Harden Windows servers per security baselines and CIS benchmarks. Configure Group Policy, Windows Defender, and security features. Use when securing Windows Server environments. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# Windows Hardening + +Secure Windows servers following Microsoft security baselines and CIS benchmarks. + +## When to Use This Skill + +Use this skill when: +- Hardening Windows servers +- Implementing security baselines +- Meeting compliance requirements +- Configuring Windows security features + +## Security Baseline + +```powershell +# Download Microsoft Security Baseline +# Apply via Group Policy or LGPO tool + +# Install Security Compliance Toolkit +Install-Module -Name SecurityPolicyDsc +``` + +## Account Policies + +```powershell +# Password policy via Group Policy +# Computer Configuration > Policies > Windows Settings > Security Settings + +# PowerShell alternative +net accounts /minpwlen:14 /maxpwage:90 /minpwage:1 /uniquepw:24 + +# Disable Administrator account +Rename-LocalUser -Name "Administrator" -NewName "LocalAdmin" +Disable-LocalUser -Name "Guest" +``` + +## Windows Firewall + +```powershell +# Enable firewall +Set-NetFirewallProfile -Profile Domain,Public,Private -Enabled True + +# Default deny +Set-NetFirewallProfile -DefaultInboundAction Block -DefaultOutboundAction Allow + +# Allow specific rules +New-NetFirewallRule -DisplayName "Allow RDP" -Direction Inbound -Protocol TCP -LocalPort 3389 -Action Allow +``` + +## Audit Configuration + +```powershell +# Enable advanced audit policy +auditpol /set /subcategory:"Logon" /success:enable /failure:enable +auditpol /set /subcategory:"Account Lockout" /success:enable /failure:enable +auditpol /set /subcategory:"Security Group Management" /success:enable + +# Enable PowerShell logging +Set-ItemProperty -Path "HKLM:\SOFTWARE\Policies\Microsoft\Windows\PowerShell\ScriptBlockLogging" -Name "EnableScriptBlockLogging" -Value 1 +``` + +## Windows Defender + +```powershell +# Enable real-time protection +Set-MpPreference -DisableRealtimeMonitoring $false + +# Enable cloud protection +Set-MpPreference -MAPSReporting Advanced + +# Configure scans +Set-MpPreference -ScanScheduleDay Everyday +Set-MpPreference -ScanScheduleTime 02:00:00 +``` + +## Best Practices + +- Apply security baselines +- Enable Windows Defender ATP +- Configure AppLocker +- Disable SMBv1 +- Enable Credential Guard +- Regular Windows updates +- Implement LAPS for local admin passwords + +## Related Skills + +- [cis-benchmarks](../cis-benchmarks/) - Compliance scanning +- [windows-server](../../../infrastructure/servers/windows-server/) - Server administration diff --git a/security/network/firewall-config/SKILL.md b/security/network/firewall-config/SKILL.md new file mode 100644 index 0000000..ccf8025 --- /dev/null +++ b/security/network/firewall-config/SKILL.md @@ -0,0 +1,84 @@ +--- +name: firewall-config +description: Configure iptables, nftables, and cloud firewalls. Implement network segmentation and traffic filtering. Use when securing network perimeters or implementing security zones. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# Firewall Configuration + +Configure host-based and cloud firewalls for network security. + +## iptables + +```bash +# Default policies +iptables -P INPUT DROP +iptables -P FORWARD DROP +iptables -P OUTPUT ACCEPT + +# Allow established connections +iptables -A INPUT -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT + +# Allow loopback +iptables -A INPUT -i lo -j ACCEPT + +# Allow SSH +iptables -A INPUT -p tcp --dport 22 -j ACCEPT + +# Allow HTTP/HTTPS +iptables -A INPUT -p tcp -m multiport --dports 80,443 -j ACCEPT + +# Save rules +iptables-save > /etc/iptables/rules.v4 +``` + +## nftables + +```bash +#!/usr/sbin/nft -f +flush ruleset + +table inet filter { + chain input { + type filter hook input priority 0; policy drop; + ct state established,related accept + iif "lo" accept + tcp dport { 22, 80, 443 } accept + } + + chain forward { + type filter hook forward priority 0; policy drop; + } + + chain output { + type filter hook output priority 0; policy accept; + } +} +``` + +## AWS Security Groups + +```bash +aws ec2 create-security-group --group-name web-sg --description "Web server SG" + +aws ec2 authorize-security-group-ingress \ + --group-name web-sg \ + --protocol tcp --port 443 \ + --cidr 0.0.0.0/0 +``` + +## Best Practices + +- Default deny policy +- Minimal rule sets +- Regular rule audits +- Log denied traffic +- Document all rules + +## Related Skills + +- [linux-hardening](../../hardening/linux-hardening/) - System security +- [aws-vpc](../../../infrastructure/cloud-aws/aws-vpc/) - AWS networking diff --git a/security/network/firewall-config/assets/iptables-rules.sh b/security/network/firewall-config/assets/iptables-rules.sh new file mode 100644 index 0000000..5f23537 --- /dev/null +++ b/security/network/firewall-config/assets/iptables-rules.sh @@ -0,0 +1,103 @@ +#!/bin/bash +# iptables Firewall Rules Template +# Customize and apply with: bash iptables-rules.sh + +set -euo pipefail + +# Flush existing rules +iptables -F +iptables -X +iptables -t nat -F +iptables -t nat -X + +# Set default policies +iptables -P INPUT DROP +iptables -P FORWARD DROP +iptables -P OUTPUT ACCEPT + +#------------------------------------------------------------------------------ +# LOOPBACK +#------------------------------------------------------------------------------ +iptables -A INPUT -i lo -j ACCEPT +iptables -A OUTPUT -o lo -j ACCEPT + +#------------------------------------------------------------------------------ +# ESTABLISHED CONNECTIONS +#------------------------------------------------------------------------------ +iptables -A INPUT -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT +iptables -A OUTPUT -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT + +#------------------------------------------------------------------------------ +# INVALID PACKETS +#------------------------------------------------------------------------------ +iptables -A INPUT -m conntrack --ctstate INVALID -j DROP + +#------------------------------------------------------------------------------ +# ICMP (Ping) +#------------------------------------------------------------------------------ +# Allow ping (optional - comment out to disable) +iptables -A INPUT -p icmp --icmp-type echo-request -m limit --limit 1/s -j ACCEPT +iptables -A INPUT -p icmp --icmp-type echo-reply -j ACCEPT + +#------------------------------------------------------------------------------ +# SSH (Rate Limited) +#------------------------------------------------------------------------------ +iptables -A INPUT -p tcp --dport 22 -m conntrack --ctstate NEW -m recent --set --name SSH +iptables -A INPUT -p tcp --dport 22 -m conntrack --ctstate NEW -m recent --update --seconds 60 --hitcount 4 --name SSH -j DROP +iptables -A INPUT -p tcp --dport 22 -j ACCEPT + +#------------------------------------------------------------------------------ +# WEB SERVICES (Uncomment as needed) +#------------------------------------------------------------------------------ +# HTTP +# iptables -A INPUT -p tcp --dport 80 -j ACCEPT + +# HTTPS +# iptables -A INPUT -p tcp --dport 443 -j ACCEPT + +#------------------------------------------------------------------------------ +# APPLICATION PORTS (Customize) +#------------------------------------------------------------------------------ +# Application (example: 8080) +# iptables -A INPUT -p tcp --dport 8080 -j ACCEPT + +# From specific network only +# iptables -A INPUT -s 10.0.0.0/8 -p tcp --dport 8080 -j ACCEPT + +#------------------------------------------------------------------------------ +# DATABASE (Internal Only) +#------------------------------------------------------------------------------ +# PostgreSQL from internal network +# iptables -A INPUT -s 10.0.0.0/8 -p tcp --dport 5432 -j ACCEPT + +# MySQL from internal network +# iptables -A INPUT -s 10.0.0.0/8 -p tcp --dport 3306 -j ACCEPT + +#------------------------------------------------------------------------------ +# MONITORING +#------------------------------------------------------------------------------ +# Prometheus metrics +# iptables -A INPUT -s 10.0.0.0/8 -p tcp --dport 9090 -j ACCEPT + +# Node exporter +# iptables -A INPUT -s 10.0.0.0/8 -p tcp --dport 9100 -j ACCEPT + +#------------------------------------------------------------------------------ +# LOGGING +#------------------------------------------------------------------------------ +# Log dropped packets (before final DROP) +iptables -A INPUT -j LOG --log-prefix "iptables-dropped: " --log-level 4 -m limit --limit 5/min + +#------------------------------------------------------------------------------ +# FINAL DROP (Implicit with policy, but explicit for clarity) +#------------------------------------------------------------------------------ +iptables -A INPUT -j DROP + +#------------------------------------------------------------------------------ +# SAVE RULES +#------------------------------------------------------------------------------ +echo "Saving rules..." +iptables-save > /etc/iptables/rules.v4 2>/dev/null || iptables-save > /tmp/iptables-rules.v4 + +echo "Firewall configured successfully!" +iptables -L -n -v diff --git a/security/network/firewall-config/references/iptables-guide.md b/security/network/firewall-config/references/iptables-guide.md new file mode 100644 index 0000000..6db64eb --- /dev/null +++ b/security/network/firewall-config/references/iptables-guide.md @@ -0,0 +1,127 @@ +# iptables Reference Guide + +## Chain Overview + +``` + PREROUTING + │ + ▼ + ┌─────────┐ + │ ROUTING │ + └────┬────┘ + │ + ┌────────────┼────────────┐ + │ │ │ + ▼ ▼ ▼ + INPUT FORWARD OUTPUT + │ │ │ + ▼ │ ▼ + Local Process │ Local Process + │ + ▼ + POSTROUTING +``` + +## Tables + +| Table | Purpose | Chains | +|-------|---------|--------| +| filter | Default, packet filtering | INPUT, FORWARD, OUTPUT | +| nat | Network Address Translation | PREROUTING, OUTPUT, POSTROUTING | +| mangle | Packet alteration | All chains | +| raw | Connection tracking exemption | PREROUTING, OUTPUT | + +## Basic Commands + +```bash +# List rules +iptables -L -n -v # All filter rules +iptables -L INPUT -n -v # INPUT chain only +iptables -t nat -L -n -v # NAT table + +# Add rules +iptables -A INPUT -p tcp --dport 22 -j ACCEPT +iptables -I INPUT 1 -p tcp --dport 80 -j ACCEPT # Insert at position 1 + +# Delete rules +iptables -D INPUT -p tcp --dport 22 -j ACCEPT +iptables -D INPUT 3 # Delete rule #3 + +# Flush rules +iptables -F # Flush all filter rules +iptables -t nat -F # Flush NAT rules + +# Set policy +iptables -P INPUT DROP +iptables -P FORWARD DROP +iptables -P OUTPUT ACCEPT +``` + +## Common Rules + +### Allow Established Connections +```bash +iptables -A INPUT -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT +``` + +### Allow Loopback +```bash +iptables -A INPUT -i lo -j ACCEPT +iptables -A OUTPUT -o lo -j ACCEPT +``` + +### Allow SSH +```bash +iptables -A INPUT -p tcp --dport 22 -j ACCEPT + +# Rate limit SSH +iptables -A INPUT -p tcp --dport 22 -m conntrack --ctstate NEW -m recent --set +iptables -A INPUT -p tcp --dport 22 -m conntrack --ctstate NEW -m recent --update --seconds 60 --hitcount 4 -j DROP +``` + +### Allow Web Traffic +```bash +iptables -A INPUT -p tcp --dport 80 -j ACCEPT +iptables -A INPUT -p tcp --dport 443 -j ACCEPT +``` + +### Allow from Specific IP/Network +```bash +iptables -A INPUT -s 192.168.1.0/24 -j ACCEPT +iptables -A INPUT -s 10.0.0.5 -p tcp --dport 5432 -j ACCEPT +``` + +### Block IP +```bash +iptables -A INPUT -s 1.2.3.4 -j DROP +``` + +### Log Dropped Packets +```bash +iptables -A INPUT -j LOG --log-prefix "IPTables-Dropped: " --log-level 4 +iptables -A INPUT -j DROP +``` + +## NAT Rules + +### SNAT (Source NAT) +```bash +iptables -t nat -A POSTROUTING -o eth0 -j MASQUERADE +``` + +### DNAT (Destination NAT / Port Forwarding) +```bash +iptables -t nat -A PREROUTING -p tcp --dport 80 -j DNAT --to-destination 192.168.1.10:8080 +``` + +## Save/Restore + +```bash +# Save rules +iptables-save > /etc/iptables/rules.v4 +ip6tables-save > /etc/iptables/rules.v6 + +# Restore rules +iptables-restore < /etc/iptables/rules.v4 +ip6tables-restore < /etc/iptables/rules.v6 +``` diff --git a/security/network/firewall-config/scripts/firewall-audit.sh b/security/network/firewall-config/scripts/firewall-audit.sh new file mode 100644 index 0000000..6cbf4aa --- /dev/null +++ b/security/network/firewall-config/scripts/firewall-audit.sh @@ -0,0 +1,91 @@ +#!/bin/bash +# Firewall Configuration Audit Script +# Usage: ./firewall-audit.sh + +set -euo pipefail + +echo "=========================================" +echo "Firewall Configuration Audit" +echo "=========================================" +echo "" + +# Detect firewall type +if command -v ufw &>/dev/null; then + FIREWALL="ufw" +elif command -v firewall-cmd &>/dev/null; then + FIREWALL="firewalld" +elif command -v nft &>/dev/null; then + FIREWALL="nftables" +else + FIREWALL="iptables" +fi + +echo "Detected Firewall: $FIREWALL" +echo "" + +case "$FIREWALL" in + ufw) + echo "UFW Status:" + echo "----------" + ufw status verbose + echo "" + echo "UFW Rules (numbered):" + echo "--------------------" + ufw status numbered + echo "" + echo "UFW Application Profiles:" + echo "------------------------" + ufw app list + ;; + + firewalld) + echo "Firewalld Status:" + echo "----------------" + firewall-cmd --state + echo "" + echo "Active Zones:" + echo "-------------" + firewall-cmd --get-active-zones + echo "" + echo "Default Zone: $(firewall-cmd --get-default-zone)" + echo "" + echo "All Zone Rules:" + echo "--------------" + for zone in $(firewall-cmd --get-zones); do + echo "--- Zone: $zone ---" + firewall-cmd --zone=$zone --list-all 2>/dev/null || true + echo "" + done + ;; + + nftables) + echo "nftables Ruleset:" + echo "----------------" + nft list ruleset + ;; + + iptables) + echo "iptables Rules (Filter):" + echo "-----------------------" + iptables -L -n -v --line-numbers + echo "" + echo "iptables Rules (NAT):" + echo "--------------------" + iptables -t nat -L -n -v --line-numbers 2>/dev/null || true + echo "" + echo "ip6tables Rules:" + echo "---------------" + ip6tables -L -n -v --line-numbers 2>/dev/null || true + ;; +esac + +echo "" +echo "=========================================" +echo "Open Ports (listening):" +echo "=========================================" +ss -tlnp 2>/dev/null || netstat -tlnp + +echo "" +echo "=========================================" +echo "Audit complete" +echo "=========================================" diff --git a/security/network/firewall-config/scripts/setup-ufw.sh b/security/network/firewall-config/scripts/setup-ufw.sh new file mode 100644 index 0000000..1cc4f61 --- /dev/null +++ b/security/network/firewall-config/scripts/setup-ufw.sh @@ -0,0 +1,80 @@ +#!/bin/bash +# UFW Firewall Setup Script +# Usage: ./setup-ufw.sh [--apply] + +set -euo pipefail + +APPLY="${1:-}" + +if [ "$APPLY" != "--apply" ]; then + echo "DRY RUN MODE - showing commands only" + echo "Run with --apply to execute" + echo "" +fi + +run_cmd() { + if [ "$APPLY" == "--apply" ]; then + eval "$1" + else + echo "[DRY RUN] $1" + fi +} + +echo "=========================================" +echo "UFW Firewall Setup" +echo "=========================================" +echo "" + +# Reset UFW +echo "Resetting UFW to defaults..." +run_cmd "ufw --force reset" + +# Set default policies +echo "" +echo "Setting default policies..." +run_cmd "ufw default deny incoming" +run_cmd "ufw default allow outgoing" + +# Essential services +echo "" +echo "Allowing essential services..." + +# SSH (rate limited) +run_cmd "ufw limit ssh comment 'SSH with rate limiting'" + +# Common services (uncomment as needed) +echo "" +echo "Common service rules (customize as needed):" + +# Web server +# run_cmd "ufw allow 80/tcp comment 'HTTP'" +# run_cmd "ufw allow 443/tcp comment 'HTTPS'" + +# Database (restrict to specific IPs) +# run_cmd "ufw allow from 10.0.0.0/8 to any port 5432 comment 'PostgreSQL from internal'" +# run_cmd "ufw allow from 10.0.0.0/8 to any port 3306 comment 'MySQL from internal'" + +# Application ports +# run_cmd "ufw allow 8080/tcp comment 'Application'" + +# Enable logging +echo "" +echo "Enabling logging..." +run_cmd "ufw logging medium" + +# Enable firewall +echo "" +echo "Enabling UFW..." +run_cmd "ufw --force enable" + +# Show status +echo "" +echo "Final status:" +if [ "$APPLY" == "--apply" ]; then + ufw status verbose +fi + +echo "" +echo "=========================================" +echo "Setup complete" +echo "=========================================" diff --git a/security/network/ssl-tls-management/SKILL.md b/security/network/ssl-tls-management/SKILL.md new file mode 100644 index 0000000..e179b2a --- /dev/null +++ b/security/network/ssl-tls-management/SKILL.md @@ -0,0 +1,96 @@ +--- +name: ssl-tls-management +description: Manage SSL/TLS certificates with Let's Encrypt and internal PKI. Configure secure HTTPS, certificate renewal, and cipher suites. Use when implementing secure communications. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# SSL/TLS Management + +Manage certificates and secure communications. + +## Let's Encrypt (Certbot) + +```bash +# Install +apt install certbot python3-certbot-nginx + +# Get certificate +certbot --nginx -d example.com -d www.example.com + +# Auto-renewal +certbot renew --dry-run +# Cron: 0 0 * * * certbot renew --quiet +``` + +## cert-manager (Kubernetes) + +```yaml +apiVersion: cert-manager.io/v1 +kind: ClusterIssuer +metadata: + name: letsencrypt-prod +spec: + acme: + server: https://acme-v02.api.letsencrypt.org/directory + email: admin@example.com + privateKeySecretRef: + name: letsencrypt-prod + solvers: + - http01: + ingress: + class: nginx +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: example-cert +spec: + secretName: example-tls + issuerRef: + name: letsencrypt-prod + kind: ClusterIssuer + dnsNames: + - example.com +``` + +## Strong Configuration + +```nginx +# nginx ssl config +ssl_protocols TLSv1.2 TLSv1.3; +ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256; +ssl_prefer_server_ciphers off; +ssl_session_timeout 1d; +ssl_session_cache shared:SSL:10m; +ssl_stapling on; +ssl_stapling_verify on; + +add_header Strict-Transport-Security "max-age=63072000" always; +``` + +## Certificate Monitoring + +```bash +# Check expiration +openssl s_client -connect example.com:443 -servername example.com 2>/dev/null | \ + openssl x509 -noout -dates + +# Check certificate chain +openssl s_client -connect example.com:443 -showcerts +``` + +## Best Practices + +- Automate renewal +- Monitor expiration +- Use strong ciphers +- Enable HSTS +- Regular security audits + +## Related Skills + +- [hashicorp-vault](../../secrets/hashicorp-vault/) - PKI management +- [waf-setup](../waf-setup/) - Web protection diff --git a/security/network/vpn-setup/SKILL.md b/security/network/vpn-setup/SKILL.md new file mode 100644 index 0000000..b23ff44 --- /dev/null +++ b/security/network/vpn-setup/SKILL.md @@ -0,0 +1,75 @@ +--- +name: vpn-setup +description: Configure WireGuard, OpenVPN, and cloud VPNs. Implement secure remote access and site-to-site connectivity. Use when setting up secure network tunnels. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# VPN Setup + +Configure secure VPN tunnels for remote access and site connectivity. + +## WireGuard + +```bash +# Generate keys +wg genkey | tee privatekey | wg pubkey > publickey + +# Server config (/etc/wireguard/wg0.conf) +[Interface] +Address = 10.0.0.1/24 +ListenPort = 51820 +PrivateKey = + +[Peer] +PublicKey = +AllowedIPs = 10.0.0.2/32 + +# Enable +wg-quick up wg0 +systemctl enable wg-quick@wg0 +``` + +## OpenVPN + +```bash +# Install +apt install openvpn easy-rsa + +# Generate certificates +cd /etc/openvpn/easy-rsa +./easyrsa init-pki +./easyrsa build-ca +./easyrsa gen-req server nopass +./easyrsa sign-req server server +./easyrsa gen-dh +``` + +## AWS Site-to-Site VPN + +```bash +aws ec2 create-vpn-gateway --type ipsec.1 +aws ec2 create-customer-gateway \ + --type ipsec.1 \ + --bgp-asn 65000 \ + --public-ip +aws ec2 create-vpn-connection \ + --type ipsec.1 \ + --customer-gateway-id cgw-xxx \ + --vpn-gateway-id vgw-xxx +``` + +## Best Practices + +- Use WireGuard for modern deployments +- Implement MFA for VPN access +- Regular key rotation +- Monitor VPN connections +- Segment VPN access by role + +## Related Skills + +- [zero-trust](../zero-trust/) - Modern access patterns +- [ssl-tls-management](../ssl-tls-management/) - Certificate management diff --git a/security/network/waf-setup/SKILL.md b/security/network/waf-setup/SKILL.md new file mode 100644 index 0000000..2cfea6b --- /dev/null +++ b/security/network/waf-setup/SKILL.md @@ -0,0 +1,79 @@ +--- +name: waf-setup +description: Deploy and tune Web Application Firewalls. Configure rules for OWASP Top 10 protection. Use when protecting web applications from common attacks. +license: MIT +metadata: + author: devops-skills + version: "1.0" +--- + +# WAF Setup + +Protect web applications with Web Application Firewalls. + +## AWS WAF + +```bash +# Create Web ACL +aws wafv2 create-web-acl \ + --name my-waf \ + --scope REGIONAL \ + --default-action Allow={} \ + --rules file://rules.json + +# Associate with ALB +aws wafv2 associate-web-acl \ + --web-acl-arn arn:aws:wafv2:... \ + --resource-arn arn:aws:elasticloadbalancing:... +``` + +## ModSecurity (nginx) + +```nginx +# nginx.conf +load_module modules/ngx_http_modsecurity_module.so; + +server { + modsecurity on; + modsecurity_rules_file /etc/nginx/modsec/main.conf; +} +``` + +```bash +# Install OWASP CRS +git clone https://github.com/coreruleset/coreruleset /etc/nginx/modsec/crs +``` + +## Cloudflare WAF + +```bash +# Enable managed rules via API +curl -X PUT "https://api.cloudflare.com/client/v4/zones/{zone}/firewall/waf/packages/{package}/rules/{rule}" \ + -H "Authorization: Bearer $TOKEN" \ + -d '{"mode":"block"}' +``` + +## Common Rules + +```yaml +protections: + - SQL Injection (SQLi) + - Cross-Site Scripting (XSS) + - Remote File Inclusion (RFI) + - Local File Inclusion (LFI) + - Command Injection + - Cross-Site Request Forgery (CSRF) +``` + +## Best Practices + +- Start in detection mode +- Tune for false positives +- Monitor blocked requests +- Regular rule updates +- Custom rules for app-specific attacks + +## Related Skills + +- [dast-scanning](../../scanning/dast-scanning/) - Web security testing +- [ssl-tls-management](../ssl-tls-management/) - HTTPS configuration diff --git a/security/network/waf-setup/references/waf-rules.md b/security/network/waf-setup/references/waf-rules.md new file mode 100644 index 0000000..1d9606e --- /dev/null +++ b/security/network/waf-setup/references/waf-rules.md @@ -0,0 +1,120 @@ +# WAF Rules Reference + +## AWS WAF + +### Managed Rules +```hcl +resource "aws_wafv2_web_acl" "main" { + name = "myapp-waf" + scope = "REGIONAL" + + default_action { + allow {} + } + + # AWS Managed Rules - Core + rule { + name = "AWSManagedRulesCommonRuleSet" + priority = 1 + override_action { none {} } + + statement { + managed_rule_group_statement { + vendor_name = "AWS" + name = "AWSManagedRulesCommonRuleSet" + } + } + visibility_config { + cloudwatch_metrics_enabled = true + metric_name = "CommonRuleSet" + sampled_requests_enabled = true + } + } + + # SQL Injection + rule { + name = "AWSManagedRulesSQLiRuleSet" + priority = 2 + override_action { none {} } + + statement { + managed_rule_group_statement { + vendor_name = "AWS" + name = "AWSManagedRulesSQLiRuleSet" + } + } + visibility_config { + cloudwatch_metrics_enabled = true + metric_name = "SQLiRuleSet" + sampled_requests_enabled = true + } + } +} +``` + +### Custom Rules +```hcl +# Rate limiting +rule { + name = "RateLimit" + priority = 0 + action { block {} } + + statement { + rate_based_statement { + limit = 2000 + aggregate_key_type = "IP" + } + } +} + +# Geo blocking +rule { + name = "GeoBlock" + priority = 3 + action { block {} } + + statement { + geo_match_statement { + country_codes = ["CN", "RU"] + } + } +} +``` + +## Cloudflare WAF + +```hcl +resource "cloudflare_ruleset" "waf" { + zone_id = var.zone_id + name = "WAF Rules" + kind = "zone" + phase = "http_request_firewall_managed" + + rules { + action = "execute" + action_parameters { + id = "efb7b8c949ac4650a09736fc376e9aee" # OWASP Core Ruleset + } + expression = "true" + } +} +``` + +## Common Attack Patterns + +| Pattern | Description | Rule | +|---------|-------------|------| +| SQLi | SQL Injection | Block `' OR 1=1`, UNION | +| XSS | Cross-Site Scripting | Block `