mirror of
https://github.com/BagelHole/DevOps-Security-Agent-Skills.git
synced 2026-08-22 12:49:53 +02:00
V2
This commit is contained in:
Toby
@@ -9,57 +9,343 @@ metadata:
|
||||
|
||||
# Linux Administration
|
||||
|
||||
Core Linux system administration skills.
|
||||
Core Linux system administration skills for managing production servers, development environments, and infrastructure hosts across Debian/Ubuntu and RHEL/CentOS distributions.
|
||||
|
||||
## When to Use
|
||||
|
||||
- Provisioning and maintaining Linux servers in any environment
|
||||
- Installing, updating, or removing software packages
|
||||
- Managing filesystems, disk usage, and mount points
|
||||
- Investigating runaway processes or high resource consumption
|
||||
- Scheduling recurring tasks with cron or systemd timers
|
||||
- Analyzing system and application logs for troubleshooting
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- Root or sudo access on the target system
|
||||
- SSH access configured (see `ssh-configuration` skill)
|
||||
- Familiarity with a terminal text editor (vim, nano)
|
||||
- Package manager available (`apt` on Debian/Ubuntu, `dnf` on RHEL 8+/Fedora)
|
||||
|
||||
## Package Management
|
||||
|
||||
```bash
|
||||
# Debian/Ubuntu
|
||||
apt update && apt upgrade -y
|
||||
apt install nginx
|
||||
apt remove nginx
|
||||
apt autoremove
|
||||
### Debian / Ubuntu (apt)
|
||||
|
||||
# RHEL/CentOS
|
||||
dnf update
|
||||
dnf install nginx
|
||||
```bash
|
||||
# Update package index and upgrade all installed packages
|
||||
apt update && apt upgrade -y
|
||||
|
||||
# Search for a package by keyword
|
||||
apt search nginx
|
||||
|
||||
# Show detailed package info including dependencies
|
||||
apt show nginx
|
||||
|
||||
# Install a specific version of a package
|
||||
apt install nginx=1.24.0-1ubuntu1
|
||||
|
||||
# Install multiple packages in one command
|
||||
apt install -y nginx certbot python3-certbot-nginx
|
||||
|
||||
# Remove a package but keep its config files
|
||||
apt remove nginx
|
||||
|
||||
# Remove a package and purge all config files
|
||||
apt purge nginx
|
||||
|
||||
# Remove unused dependency packages
|
||||
apt autoremove -y
|
||||
|
||||
# List all installed packages
|
||||
dpkg -l | grep nginx
|
||||
|
||||
# Pin a package to prevent automatic upgrades
|
||||
cat <<'EOF' > /etc/apt/preferences.d/pin-nginx
|
||||
Package: nginx
|
||||
Pin: version 1.24.0-1ubuntu1
|
||||
Pin-Priority: 1001
|
||||
EOF
|
||||
|
||||
# Add an external repository (example: Docker CE)
|
||||
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
|
||||
echo "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" \
|
||||
> /etc/apt/sources.list.d/docker.list
|
||||
apt update
|
||||
```
|
||||
|
||||
### RHEL / CentOS / Fedora (dnf)
|
||||
|
||||
```bash
|
||||
# Update all packages
|
||||
dnf update -y
|
||||
|
||||
# Search for a package
|
||||
dnf search nginx
|
||||
|
||||
# Show package details
|
||||
dnf info nginx
|
||||
|
||||
# Install a package
|
||||
dnf install -y nginx
|
||||
|
||||
# Install a specific version
|
||||
dnf install nginx-1.24.0-1.el9
|
||||
|
||||
# Remove a package
|
||||
dnf remove nginx
|
||||
|
||||
# List installed packages
|
||||
dnf list installed | grep nginx
|
||||
|
||||
# Enable a module stream (example: Node.js 20)
|
||||
dnf module enable nodejs:20
|
||||
dnf install -y nodejs
|
||||
|
||||
# Add an external repository
|
||||
dnf install -y epel-release
|
||||
|
||||
# View repository list
|
||||
dnf repolist --all
|
||||
|
||||
# Clean cached package data
|
||||
dnf clean all
|
||||
```
|
||||
|
||||
## System Information
|
||||
|
||||
```bash
|
||||
uname -a # Kernel info
|
||||
hostnamectl # System info
|
||||
lscpu # CPU info
|
||||
free -h # Memory usage
|
||||
df -h # Disk usage
|
||||
ip addr # Network interfaces
|
||||
# Kernel and OS release
|
||||
uname -a
|
||||
cat /etc/os-release
|
||||
|
||||
# Hostname and system metadata
|
||||
hostnamectl
|
||||
|
||||
# CPU information
|
||||
lscpu
|
||||
nproc # Number of processing units
|
||||
|
||||
# Memory usage (human-readable)
|
||||
free -h
|
||||
|
||||
# Disk usage summary
|
||||
df -hT # Include filesystem type
|
||||
du -sh /var/log/* # Summarize directory sizes
|
||||
|
||||
# Network interfaces and IP addresses
|
||||
ip addr show
|
||||
ip route show # Routing table
|
||||
|
||||
# Uptime and load average
|
||||
uptime
|
||||
w # Who is logged in and load
|
||||
```
|
||||
|
||||
## Log Management
|
||||
## Filesystem Management
|
||||
|
||||
```bash
|
||||
journalctl -u nginx # Service logs
|
||||
journalctl -f # Follow logs
|
||||
tail -f /var/log/syslog # System logs
|
||||
dmesg # Kernel messages
|
||||
# List block devices and partitions
|
||||
lsblk
|
||||
fdisk -l
|
||||
|
||||
# Create a new ext4 filesystem on a partition
|
||||
mkfs.ext4 /dev/sdb1
|
||||
|
||||
# Mount a filesystem temporarily
|
||||
mount /dev/sdb1 /mnt/data
|
||||
|
||||
# Add a persistent mount via fstab
|
||||
echo '/dev/sdb1 /mnt/data ext4 defaults,noatime 0 2' >> /etc/fstab
|
||||
mount -a # Mount everything in fstab
|
||||
|
||||
# Check and repair a filesystem (unmount first)
|
||||
umount /dev/sdb1
|
||||
fsck.ext4 -y /dev/sdb1
|
||||
|
||||
# Monitor disk I/O in real time
|
||||
iostat -xz 2
|
||||
|
||||
# Find files larger than 100 MB
|
||||
find / -xdev -type f -size +100M -exec ls -lh {} \;
|
||||
|
||||
# Check inode usage (out-of-inodes can mimic out-of-disk)
|
||||
df -i
|
||||
```
|
||||
|
||||
## Process Management
|
||||
|
||||
```bash
|
||||
ps aux | grep nginx
|
||||
top / htop
|
||||
# List all processes with full details
|
||||
ps auxf
|
||||
|
||||
# Interactive process viewer (prefer htop if installed)
|
||||
top
|
||||
htop
|
||||
|
||||
# Find processes by name
|
||||
pgrep -la nginx
|
||||
|
||||
# Show process tree
|
||||
pstree -p
|
||||
|
||||
# Send graceful stop signal (SIGTERM)
|
||||
kill <pid>
|
||||
|
||||
# Force kill an unresponsive process (SIGKILL)
|
||||
kill -9 <pid>
|
||||
pgrep nginx
|
||||
|
||||
# Kill all processes matching a name
|
||||
pkill nginx
|
||||
|
||||
# Show open files for a process
|
||||
lsof -p <pid>
|
||||
|
||||
# Show which process is listening on a port
|
||||
ss -tlnp | grep :80
|
||||
lsof -i :80
|
||||
|
||||
# Run a process immune to hangups (persists after logout)
|
||||
nohup /opt/myapp/start.sh > /var/log/myapp.log 2>&1 &
|
||||
|
||||
# Limit CPU usage of a running process with cgroups v2
|
||||
systemd-run --scope -p CPUQuota=25% --unit=limit-myapp /opt/myapp/start.sh
|
||||
```
|
||||
|
||||
## Best Practices
|
||||
## Cron Job Management
|
||||
|
||||
- Regular updates
|
||||
- Minimal installed packages
|
||||
- Proper file permissions
|
||||
- Log rotation configuration
|
||||
- Automated backups
|
||||
```bash
|
||||
# Edit the current user's crontab
|
||||
crontab -e
|
||||
|
||||
# List current user's cron jobs
|
||||
crontab -l
|
||||
|
||||
# Example crontab entries
|
||||
# ┌───── minute (0-59)
|
||||
# │ ┌───── hour (0-23)
|
||||
# │ │ ┌───── day of month (1-31)
|
||||
# │ │ │ ┌───── month (1-12)
|
||||
# │ │ │ │ ┌───── day of week (0-7, 0 and 7 = Sunday)
|
||||
# * * * * * command
|
||||
|
||||
# Run a backup every day at 2:30 AM
|
||||
30 2 * * * /usr/local/bin/backup.sh >> /var/log/backup.log 2>&1
|
||||
|
||||
# Run a cleanup every Sunday at midnight
|
||||
0 0 * * 0 /usr/local/bin/cleanup.sh
|
||||
|
||||
# Run a health check every 5 minutes
|
||||
*/5 * * * * /usr/local/bin/healthcheck.sh
|
||||
|
||||
# Place system-wide cron scripts in drop-in directories
|
||||
ls /etc/cron.daily/
|
||||
ls /etc/cron.weekly/
|
||||
|
||||
# Restrict cron access to specific users
|
||||
echo "deploy" >> /etc/cron.allow
|
||||
```
|
||||
|
||||
## Log Management
|
||||
|
||||
```bash
|
||||
# Follow systemd journal for a specific service
|
||||
journalctl -u nginx -f
|
||||
|
||||
# Show logs since last boot
|
||||
journalctl -b
|
||||
|
||||
# Show logs from a specific time range
|
||||
journalctl --since "2025-01-15 08:00" --until "2025-01-15 12:00"
|
||||
|
||||
# Show only error-level and above
|
||||
journalctl -p err
|
||||
|
||||
# Tail traditional syslog
|
||||
tail -f /var/log/syslog # Debian/Ubuntu
|
||||
tail -f /var/log/messages # RHEL/CentOS
|
||||
|
||||
# Kernel ring buffer messages
|
||||
dmesg -T # Human-readable timestamps
|
||||
dmesg --level=err,warn
|
||||
|
||||
# Check disk usage of log directory
|
||||
du -sh /var/log/*
|
||||
|
||||
# Configure logrotate for a custom application
|
||||
cat <<'EOF' > /etc/logrotate.d/myapp
|
||||
/var/log/myapp/*.log {
|
||||
daily
|
||||
missingok
|
||||
rotate 14
|
||||
compress
|
||||
delaycompress
|
||||
notifempty
|
||||
create 0640 myapp myapp
|
||||
sharedscripts
|
||||
postrotate
|
||||
systemctl reload myapp > /dev/null 2>&1 || true
|
||||
endscript
|
||||
}
|
||||
EOF
|
||||
|
||||
# Force a logrotate run for testing
|
||||
logrotate -f /etc/logrotate.d/myapp
|
||||
|
||||
# Centralized logging: forward journal to a remote syslog
|
||||
# In /etc/systemd/journal-upload.conf:
|
||||
# URL=http://logserver.example.com:19532
|
||||
```
|
||||
|
||||
## Networking Essentials
|
||||
|
||||
```bash
|
||||
# Test connectivity
|
||||
ping -c 4 8.8.8.8
|
||||
|
||||
# DNS lookup
|
||||
dig example.com
|
||||
nslookup example.com
|
||||
|
||||
# Trace route to host
|
||||
traceroute example.com
|
||||
|
||||
# List listening ports and associated processes
|
||||
ss -tlnp
|
||||
|
||||
# Show active connections
|
||||
ss -tunap
|
||||
|
||||
# Firewall management (UFW on Ubuntu)
|
||||
ufw allow 22/tcp
|
||||
ufw allow 80/tcp
|
||||
ufw allow 443/tcp
|
||||
ufw enable
|
||||
ufw status verbose
|
||||
|
||||
# Firewall management (firewalld on RHEL/CentOS)
|
||||
firewall-cmd --permanent --add-service=http
|
||||
firewall-cmd --permanent --add-service=https
|
||||
firewall-cmd --reload
|
||||
firewall-cmd --list-all
|
||||
```
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
| Symptom | Diagnostic Command | Common Fix |
|
||||
|---|---|---|
|
||||
| Disk full | `df -h` and `du -sh /var/log/*` | Clear old logs, run `logrotate -f`, remove temp files |
|
||||
| Out of inodes | `df -i` | Delete many small files, check `/tmp` and mail spools |
|
||||
| High CPU usage | `top`, `ps aux --sort=-%cpu` | Identify and restart or kill the offending process |
|
||||
| High memory / swapping | `free -h`, `vmstat 1` | Tune `vm.swappiness`, add RAM, identify memory leak |
|
||||
| Service won't start | `systemctl status <svc>`, `journalctl -u <svc>` | Check config syntax, file permissions, port conflicts |
|
||||
| DNS resolution fails | `dig @8.8.8.8 example.com`, `cat /etc/resolv.conf` | Fix nameserver entries, restart `systemd-resolved` |
|
||||
| Package dependency error | `apt --fix-broken install` or `dnf distro-sync` | Resolve held or conflicting packages |
|
||||
| SSH connection refused | `ss -tlnp \| grep 22`, `systemctl status sshd` | Ensure sshd is running and firewall allows port 22 |
|
||||
|
||||
## Related Skills
|
||||
|
||||
- `ssh-configuration` -- Secure remote access to Linux servers
|
||||
- `user-management` -- Create and manage users, groups, and sudo
|
||||
- `systemd-services` -- Write and manage systemd unit files
|
||||
- `performance-tuning` -- Kernel and application performance optimization
|
||||
- `backup-recovery` -- Protect server data with automated backups
|
||||
|
||||
@@ -9,61 +9,357 @@ metadata:
|
||||
|
||||
# Performance Tuning
|
||||
|
||||
Optimize Linux system performance.
|
||||
Optimize Linux system performance through kernel parameter tuning, I/O scheduler selection, memory management, CPU governor configuration, and benchmarking. Covers methodology, real sysctl settings, and tool-based validation.
|
||||
|
||||
## System Monitoring
|
||||
## When to Use
|
||||
|
||||
- Server experiencing high latency, throughput bottlenecks, or resource exhaustion
|
||||
- Preparing infrastructure for high-traffic events or load tests
|
||||
- Tuning a database server, web server, or application host for production
|
||||
- Diagnosing whether a bottleneck is CPU, memory, disk I/O, or network
|
||||
- Establishing baseline performance metrics before and after changes
|
||||
- Configuring kernel parameters for containers, VMs, or bare-metal hosts
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- Root or sudo access on the target system
|
||||
- `sysstat` package installed (provides `sar`, `iostat`, `mpstat`)
|
||||
- `linux-tools` or `perf` package for CPU profiling
|
||||
- Benchmarking tools: `fio` (disk), `sysbench` (CPU/memory), `iperf3` (network)
|
||||
- Baseline metrics collected before making any changes
|
||||
|
||||
## Performance Analysis Methodology
|
||||
|
||||
Always follow this order:
|
||||
|
||||
1. **Collect baseline** -- measure current performance with tools
|
||||
2. **Identify bottleneck** -- determine if CPU, memory, I/O, or network
|
||||
3. **Change one parameter** -- apply a single tuning change
|
||||
4. **Measure impact** -- re-run the same benchmark
|
||||
5. **Document** -- record the change and its effect
|
||||
6. **Iterate or revert** -- keep the change if beneficial, revert if not
|
||||
|
||||
## System Monitoring Tools
|
||||
|
||||
```bash
|
||||
top / htop # Process monitoring
|
||||
vmstat 1 # Memory statistics
|
||||
iostat -x 1 # Disk I/O
|
||||
sar -n DEV 1 # Network statistics
|
||||
perf top # CPU profiling
|
||||
# CPU and process monitoring
|
||||
top # Interactive process viewer
|
||||
htop # Enhanced interactive viewer
|
||||
mpstat -P ALL 2 # Per-CPU utilization every 2 seconds
|
||||
pidstat -u 2 # Per-process CPU usage
|
||||
|
||||
# Memory monitoring
|
||||
free -h # Memory summary
|
||||
vmstat 2 # Virtual memory stats every 2 seconds
|
||||
# Columns: r=runnable, b=blocked, si/so=swap in/out, bi/bo=block I/O
|
||||
|
||||
# Disk I/O monitoring
|
||||
iostat -xz 2 # Extended disk stats every 2 seconds
|
||||
# Key columns: %util, await (latency), r/s, w/s
|
||||
iotop -oP # Show processes doing I/O
|
||||
|
||||
# Network monitoring
|
||||
sar -n DEV 2 # Network interface stats
|
||||
ss -s # Socket summary
|
||||
nstat # Network counters
|
||||
|
||||
# CPU profiling (requires perf)
|
||||
perf top # Real-time function-level CPU profiling
|
||||
perf stat -a sleep 10 # System-wide counters for 10 seconds
|
||||
perf record -g -a sleep 30 # Record 30 seconds of call stacks
|
||||
perf report # Analyze recorded data
|
||||
|
||||
# One-liner: check all major resources
|
||||
echo "=== CPU ===" && mpstat 1 1 && echo "=== MEM ===" && free -h && echo "=== DISK ===" && iostat -x 1 1 && echo "=== NET ===" && ss -s
|
||||
```
|
||||
|
||||
## Kernel Parameters
|
||||
## Sysctl Kernel Parameter Tuning
|
||||
|
||||
### Network Tuning
|
||||
|
||||
```bash
|
||||
# /etc/sysctl.d/99-performance.conf
|
||||
vm.swappiness = 10
|
||||
# /etc/sysctl.d/60-network-performance.conf
|
||||
|
||||
# Increase the maximum socket receive/send buffer sizes
|
||||
net.core.rmem_max = 134217728
|
||||
net.core.wmem_max = 134217728
|
||||
net.core.rmem_default = 1048576
|
||||
net.core.wmem_default = 1048576
|
||||
|
||||
# TCP buffer auto-tuning (min, default, max in bytes)
|
||||
net.ipv4.tcp_rmem = 4096 1048576 134217728
|
||||
net.ipv4.tcp_wmem = 4096 1048576 134217728
|
||||
|
||||
# Increase connection backlog for high-traffic servers
|
||||
net.core.somaxconn = 65535
|
||||
net.ipv4.tcp_max_syn_backlog = 65535
|
||||
net.core.netdev_max_backlog = 65535
|
||||
|
||||
# Enable TCP fast open (client and server)
|
||||
net.ipv4.tcp_fastopen = 3
|
||||
|
||||
# Reuse TIME_WAIT sockets for new connections
|
||||
net.ipv4.tcp_tw_reuse = 1
|
||||
|
||||
# Increase the range of ephemeral ports
|
||||
net.ipv4.ip_local_port_range = 1024 65535
|
||||
|
||||
# TCP keepalive tuning (detect dead connections faster)
|
||||
net.ipv4.tcp_keepalive_time = 120
|
||||
net.ipv4.tcp_keepalive_intvl = 30
|
||||
net.ipv4.tcp_keepalive_probes = 3
|
||||
|
||||
# Disable slow start after idle (keeps congestion window open)
|
||||
net.ipv4.tcp_slow_start_after_idle = 0
|
||||
|
||||
# Enable BBR congestion control (requires kernel 4.9+)
|
||||
net.core.default_qdisc = fq
|
||||
net.ipv4.tcp_congestion_control = bbr
|
||||
```
|
||||
|
||||
### Memory Tuning
|
||||
|
||||
```bash
|
||||
# /etc/sysctl.d/60-memory-performance.conf
|
||||
|
||||
# Reduce swappiness (0-100, lower = less swap usage)
|
||||
# 10 for general servers, 1 for database servers
|
||||
vm.swappiness = 10
|
||||
|
||||
# Dirty page ratios (controls when dirty data is flushed to disk)
|
||||
# Lower values = more frequent, smaller writes (better for SSDs)
|
||||
vm.dirty_ratio = 20
|
||||
vm.dirty_background_ratio = 5
|
||||
|
||||
# For large-memory systems writing to fast storage
|
||||
# vm.dirty_ratio = 40
|
||||
# vm.dirty_background_ratio = 10
|
||||
|
||||
# Increase inotify limits (for apps watching many files)
|
||||
fs.inotify.max_user_watches = 524288
|
||||
fs.inotify.max_user_instances = 1024
|
||||
|
||||
# Maximum number of open file descriptors system-wide
|
||||
fs.file-max = 2097152
|
||||
vm.dirty_ratio = 40
|
||||
vm.dirty_background_ratio = 10
|
||||
|
||||
# Virtual memory overcommit
|
||||
# 0 = heuristic (default), 1 = always overcommit, 2 = never overcommit
|
||||
vm.overcommit_memory = 0
|
||||
|
||||
# For Redis or similar in-memory stores, use:
|
||||
# vm.overcommit_memory = 1
|
||||
|
||||
# Disable Transparent Huge Pages if it causes latency spikes (common with databases)
|
||||
# Done via boot parameter or runtime:
|
||||
# echo madvise > /sys/kernel/mm/transparent_hugepage/enabled
|
||||
```
|
||||
|
||||
## File Descriptor Limits
|
||||
### Applying Sysctl Changes
|
||||
|
||||
```bash
|
||||
# /etc/security/limits.conf
|
||||
* soft nofile 65535
|
||||
* hard nofile 65535
|
||||
* soft nproc 65535
|
||||
* hard nproc 65535
|
||||
# Apply all sysctl files
|
||||
sysctl --system
|
||||
|
||||
# Apply a specific file
|
||||
sysctl -p /etc/sysctl.d/60-network-performance.conf
|
||||
|
||||
# Set a parameter temporarily (lost on reboot)
|
||||
sysctl -w vm.swappiness=10
|
||||
|
||||
# Verify a parameter
|
||||
sysctl vm.swappiness
|
||||
sysctl net.ipv4.tcp_congestion_control
|
||||
```
|
||||
|
||||
## Disk I/O
|
||||
## I/O Scheduler Configuration
|
||||
|
||||
```bash
|
||||
# Change scheduler
|
||||
echo noop > /sys/block/sda/queue/scheduler
|
||||
# Check the current scheduler for a device
|
||||
cat /sys/block/sda/queue/scheduler
|
||||
# Output example: [mq-deadline] none kyber bfq
|
||||
|
||||
# Enable trim for SSDs
|
||||
fstrim -av
|
||||
# Set the scheduler temporarily
|
||||
echo mq-deadline > /sys/block/sda/queue/scheduler # Good for databases
|
||||
echo none > /sys/block/nvme0n1/queue/scheduler # Best for NVMe SSDs
|
||||
echo bfq > /sys/block/sda/queue/scheduler # Good for interactive desktop
|
||||
|
||||
# Scheduler recommendations:
|
||||
# NVMe SSD: none (noop) -- minimal overhead, hardware handles scheduling
|
||||
# SATA SSD: mq-deadline -- provides fairness with low latency
|
||||
# HDD: mq-deadline -- prevents starvation, good for databases
|
||||
# Desktop: bfq -- prioritizes interactive I/O
|
||||
|
||||
# Make persistent via udev rule
|
||||
cat <<'EOF' > /etc/udev/rules.d/60-io-scheduler.rules
|
||||
# Set mq-deadline for rotational (HDD) devices
|
||||
ACTION=="add|change", KERNEL=="sd[a-z]", ATTR{queue/rotational}=="1", ATTR{queue/scheduler}="mq-deadline"
|
||||
# Set none for non-rotational (SSD/NVMe) devices
|
||||
ACTION=="add|change", KERNEL=="sd[a-z]", ATTR{queue/rotational}=="0", ATTR{queue/scheduler}="none"
|
||||
ACTION=="add|change", KERNEL=="nvme[0-9]*", ATTR{queue/scheduler}="none"
|
||||
EOF
|
||||
|
||||
udevadm control --reload-rules
|
||||
|
||||
# Tune read-ahead for sequential workloads (database sequential scans)
|
||||
blockdev --setrahead 4096 /dev/sda # 4096 sectors = 2 MB
|
||||
|
||||
# Enable TRIM for SSDs (weekly via systemd timer)
|
||||
systemctl enable --now fstrim.timer
|
||||
fstrim -av # Manual run
|
||||
```
|
||||
|
||||
## Network Tuning
|
||||
## CPU Governor Configuration
|
||||
|
||||
```bash
|
||||
# Increase buffers
|
||||
sysctl -w net.core.rmem_max=134217728
|
||||
sysctl -w net.core.wmem_max=134217728
|
||||
# Check available governors
|
||||
cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_available_governors
|
||||
# Output: performance powersave schedutil
|
||||
|
||||
# Check current governor
|
||||
cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor
|
||||
|
||||
# Set all CPUs to performance mode (maximum frequency)
|
||||
for cpu in /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor; do
|
||||
echo performance > "$cpu"
|
||||
done
|
||||
|
||||
# Set using cpupower (if installed)
|
||||
cpupower frequency-set -g performance
|
||||
|
||||
# Governor recommendations:
|
||||
# Server (production): performance -- max frequency, lowest latency
|
||||
# Server (general): schedutil -- kernel-driven dynamic scaling
|
||||
# Laptop / idle server: powersave -- minimize power consumption
|
||||
|
||||
# Make persistent via systemd service
|
||||
cat <<'EOF' > /etc/systemd/system/cpu-governor.service
|
||||
[Unit]
|
||||
Description=Set CPU governor to performance
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/bin/cpupower frequency-set -g performance
|
||||
RemainAfterExit=yes
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
EOF
|
||||
|
||||
systemctl enable --now cpu-governor
|
||||
|
||||
# Disable CPU boost (turbo) if consistent latency is needed
|
||||
echo 1 > /sys/devices/system/cpu/intel_pstate/no_turbo
|
||||
```
|
||||
|
||||
## Best Practices
|
||||
## Benchmarking Tools
|
||||
|
||||
- Profile before optimizing
|
||||
- Change one parameter at a time
|
||||
- Monitor impact of changes
|
||||
- Document all tuning
|
||||
### fio -- Disk I/O Benchmarking
|
||||
|
||||
```bash
|
||||
# Install fio
|
||||
apt install -y fio # Debian/Ubuntu
|
||||
dnf install -y fio # RHEL/CentOS
|
||||
|
||||
# Sequential read test (simulates backup reads)
|
||||
fio --name=seq-read --ioengine=libaio --direct=1 --rw=read \
|
||||
--bs=1M --numjobs=4 --size=1G --runtime=60 --time_based --group_reporting
|
||||
|
||||
# Sequential write test
|
||||
fio --name=seq-write --ioengine=libaio --direct=1 --rw=write \
|
||||
--bs=1M --numjobs=4 --size=1G --runtime=60 --time_based --group_reporting
|
||||
|
||||
# Random read (4K blocks -- simulates database IOPS)
|
||||
fio --name=rand-read --ioengine=libaio --direct=1 --rw=randread \
|
||||
--bs=4k --numjobs=16 --iodepth=64 --size=1G --runtime=60 --time_based --group_reporting
|
||||
|
||||
# Random write (4K blocks)
|
||||
fio --name=rand-write --ioengine=libaio --direct=1 --rw=randwrite \
|
||||
--bs=4k --numjobs=16 --iodepth=64 --size=1G --runtime=60 --time_based --group_reporting
|
||||
|
||||
# Mixed random read/write (70/30 -- typical database workload)
|
||||
fio --name=mixed --ioengine=libaio --direct=1 --rw=randrw --rwmixread=70 \
|
||||
--bs=4k --numjobs=8 --iodepth=32 --size=1G --runtime=60 --time_based --group_reporting
|
||||
```
|
||||
|
||||
### sysbench -- CPU and Memory Benchmarking
|
||||
|
||||
```bash
|
||||
# Install: apt install -y sysbench (Debian) / dnf install -y sysbench (RHEL)
|
||||
|
||||
# CPU benchmark
|
||||
sysbench cpu --threads=4 --time=30 run
|
||||
|
||||
# Memory benchmark
|
||||
sysbench memory --threads=4 --time=30 --memory-block-size=1K --memory-total-size=100G run
|
||||
```
|
||||
|
||||
### iperf3 -- Network Benchmarking
|
||||
|
||||
```bash
|
||||
# Install iperf3
|
||||
apt install -y iperf3
|
||||
|
||||
# Start server on one host
|
||||
iperf3 -s
|
||||
|
||||
# Run client test from another host
|
||||
iperf3 -c <server-ip> -t 30 -P 4 # 30 seconds, 4 parallel streams
|
||||
|
||||
# Test with UDP (measure packet loss)
|
||||
iperf3 -c <server-ip> -u -b 1G -t 30
|
||||
|
||||
# Reverse mode (server sends to client)
|
||||
iperf3 -c <server-ip> -R -t 30
|
||||
```
|
||||
|
||||
## Quick-Reference Tuning Profiles
|
||||
|
||||
### Web Server (nginx/Apache)
|
||||
|
||||
```bash
|
||||
# /etc/sysctl.d/60-webserver.conf
|
||||
net.core.somaxconn = 65535
|
||||
net.ipv4.tcp_max_syn_backlog = 65535
|
||||
net.ipv4.tcp_tw_reuse = 1
|
||||
net.ipv4.tcp_fastopen = 3
|
||||
net.ipv4.ip_local_port_range = 1024 65535
|
||||
net.core.default_qdisc = fq
|
||||
net.ipv4.tcp_congestion_control = bbr
|
||||
fs.file-max = 2097152
|
||||
vm.swappiness = 10
|
||||
```
|
||||
|
||||
### Database Server (PostgreSQL/MySQL)
|
||||
|
||||
```bash
|
||||
# /etc/sysctl.d/60-database.conf
|
||||
vm.swappiness = 1
|
||||
vm.dirty_ratio = 15
|
||||
vm.dirty_background_ratio = 3
|
||||
vm.overcommit_memory = 2
|
||||
vm.overcommit_ratio = 80
|
||||
net.core.somaxconn = 4096
|
||||
fs.file-max = 2097152
|
||||
# Disable THP for databases
|
||||
# echo never > /sys/kernel/mm/transparent_hugepage/enabled
|
||||
```
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
| Symptom | Diagnostic Command | Common Fix |
|
||||
|---|---|---|
|
||||
| High CPU, no obvious process | `perf top`, `mpstat -P ALL 2` | Check for kernel-level issues: softirqs, interrupts |
|
||||
| High load avg, low CPU usage | `vmstat 2` (check `b` column) | I/O bottleneck: tune scheduler, check disk health |
|
||||
| System swapping heavily | `free -h`, `vmstat 2` (check si/so) | Reduce `vm.swappiness`, add RAM, find memory leak |
|
||||
| Disk latency spikes | `iostat -x 2` (check await) | Switch I/O scheduler, reduce dirty ratio, add SSD |
|
||||
| "Too many open files" error | `cat /proc/sys/fs/file-nr` | Increase `fs.file-max` and `LimitNOFILE` |
|
||||
| Network throughput low | `iperf3 -c <server>`, `ethtool` | Increase buffer sizes, enable BBR, check MTU |
|
||||
| Application timeout under load | `ss -s`, `sysctl net.core.somaxconn` | Increase `somaxconn` and `tcp_max_syn_backlog` |
|
||||
| Inconsistent latency | Check CPU governor | Set governor to `performance`, disable turbo boost |
|
||||
|
||||
## Related Skills
|
||||
|
||||
- `linux-administration` -- General system monitoring and management
|
||||
- `systemd-services` -- Resource limits via cgroups in unit files
|
||||
- `block-storage` -- Storage-level performance (LVM, RAID, filesystems)
|
||||
- `nfs-storage` -- NFS-specific performance tuning
|
||||
|
||||
@@ -9,68 +9,300 @@ metadata:
|
||||
|
||||
# SSH Configuration
|
||||
|
||||
Secure SSH server and client configuration.
|
||||
Secure SSH server and client configuration for production environments, including key management, hardened sshd settings, bastion host architecture, tunneling, and multiplexing.
|
||||
|
||||
## Key Management
|
||||
## When to Use
|
||||
|
||||
- Setting up secure remote access to Linux or Unix servers
|
||||
- Hardening SSH daemon configuration to meet compliance requirements
|
||||
- Configuring bastion / jump hosts for private network access
|
||||
- Creating SSH tunnels for secure port forwarding
|
||||
- Managing SSH keys for teams or automated deployments
|
||||
- Troubleshooting connection, authentication, or performance issues
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- OpenSSH client installed locally (`ssh -V` to verify)
|
||||
- OpenSSH server installed on target (`sshd`)
|
||||
- Root or sudo access on the server for sshd_config changes
|
||||
- Firewall rules allowing TCP port 22 (or custom SSH port)
|
||||
|
||||
## Key Generation and Management
|
||||
|
||||
```bash
|
||||
# Generate key
|
||||
ssh-keygen -t ed25519 -C "user@example.com"
|
||||
# Generate an Ed25519 key (recommended -- fast, secure, short)
|
||||
ssh-keygen -t ed25519 -C "jane@example.com" -f ~/.ssh/id_ed25519
|
||||
|
||||
# Copy to server
|
||||
ssh-copy-id user@server
|
||||
# Generate an RSA 4096-bit key (for legacy compatibility)
|
||||
ssh-keygen -t rsa -b 4096 -C "jane@example.com" -f ~/.ssh/id_rsa_legacy
|
||||
|
||||
# Add to agent
|
||||
# Generate a key with a custom comment and no passphrase (CI/CD use only)
|
||||
ssh-keygen -t ed25519 -C "ci-deploy-key" -f ~/.ssh/ci_deploy -N ""
|
||||
|
||||
# Copy public key to a remote server
|
||||
ssh-copy-id -i ~/.ssh/id_ed25519.pub user@server
|
||||
|
||||
# Manually append a public key (when ssh-copy-id is unavailable)
|
||||
cat ~/.ssh/id_ed25519.pub | ssh user@server "mkdir -p ~/.ssh && chmod 700 ~/.ssh && cat >> ~/.ssh/authorized_keys && chmod 600 ~/.ssh/authorized_keys"
|
||||
|
||||
# List fingerprints of keys on the agent
|
||||
ssh-add -l
|
||||
|
||||
# Start the SSH agent and add a key
|
||||
eval "$(ssh-agent -s)"
|
||||
ssh-add ~/.ssh/id_ed25519
|
||||
|
||||
# Add a key with a lifetime (auto-removed after 8 hours)
|
||||
ssh-add -t 28800 ~/.ssh/id_ed25519
|
||||
|
||||
# Remove all keys from the agent
|
||||
ssh-add -D
|
||||
|
||||
# Convert an OpenSSH key to PEM format (for tools that need it)
|
||||
ssh-keygen -p -m PEM -f ~/.ssh/id_rsa_legacy
|
||||
|
||||
# Show the public key fingerprint (SHA256)
|
||||
ssh-keygen -lf ~/.ssh/id_ed25519.pub
|
||||
|
||||
# Rotate a key: generate new, deploy, then revoke old
|
||||
ssh-keygen -t ed25519 -f ~/.ssh/id_ed25519_new -C "jane@example.com rotated $(date +%Y-%m)"
|
||||
ssh-copy-id -i ~/.ssh/id_ed25519_new.pub user@server
|
||||
# After verifying the new key works, remove the old public key from authorized_keys on the server
|
||||
```
|
||||
|
||||
## SSH Config (~/.ssh/config)
|
||||
## SSH Client Configuration (~/.ssh/config)
|
||||
|
||||
```
|
||||
Host production
|
||||
HostName prod.example.com
|
||||
User deploy
|
||||
IdentityFile ~/.ssh/prod_key
|
||||
Port 22
|
||||
```text
|
||||
# Global defaults applied to all hosts
|
||||
Host *
|
||||
AddKeysToAgent yes
|
||||
IdentitiesOnly yes
|
||||
ServerAliveInterval 60
|
||||
ServerAliveCountMax 3
|
||||
TCPKeepAlive yes
|
||||
Compression yes
|
||||
|
||||
# Production servers via bastion
|
||||
Host bastion
|
||||
HostName bastion.example.com
|
||||
User admin
|
||||
|
||||
Host internal
|
||||
HostName 10.0.0.5
|
||||
User admin
|
||||
User ops
|
||||
IdentityFile ~/.ssh/id_ed25519
|
||||
Port 22
|
||||
|
||||
Host prod-web-*
|
||||
User deploy
|
||||
IdentityFile ~/.ssh/id_ed25519
|
||||
ProxyJump bastion
|
||||
Port 22
|
||||
|
||||
Host prod-web-1
|
||||
HostName 10.0.1.10
|
||||
|
||||
Host prod-web-2
|
||||
HostName 10.0.1.11
|
||||
|
||||
# Staging accessed directly
|
||||
Host staging
|
||||
HostName staging.example.com
|
||||
User deploy
|
||||
IdentityFile ~/.ssh/id_ed25519_staging
|
||||
|
||||
# Database tunnel through bastion
|
||||
Host db-tunnel
|
||||
HostName 10.0.2.50
|
||||
User dba
|
||||
ProxyJump bastion
|
||||
LocalForward 5432 localhost:5432
|
||||
|
||||
# GitHub deploy key
|
||||
Host github-deploy
|
||||
HostName github.com
|
||||
User git
|
||||
IdentityFile ~/.ssh/github_deploy_key
|
||||
IdentitiesOnly yes
|
||||
|
||||
# Connection multiplexing for faster repeated connections
|
||||
Host fast-*
|
||||
ControlMaster auto
|
||||
ControlPath ~/.ssh/sockets/%r@%h-%p
|
||||
ControlPersist 600
|
||||
```
|
||||
|
||||
## Secure Server Config
|
||||
```bash
|
||||
# Create the sockets directory for multiplexing
|
||||
mkdir -p ~/.ssh/sockets
|
||||
chmod 700 ~/.ssh/sockets
|
||||
```
|
||||
|
||||
## Hardened Server Configuration (/etc/ssh/sshd_config)
|
||||
|
||||
```bash
|
||||
# /etc/ssh/sshd_config
|
||||
# /etc/ssh/sshd_config -- hardened configuration
|
||||
# -----------------------------------------------
|
||||
|
||||
# Listen on a non-default port (obscurity, not security -- combine with firewall)
|
||||
Port 22
|
||||
|
||||
# Protocol and key exchange
|
||||
Protocol 2
|
||||
KexAlgorithms curve25519-sha256,curve25519-sha256@libssh.org,diffie-hellman-group16-sha512
|
||||
Ciphers chacha20-poly1305@openssh.com,aes256-gcm@openssh.com,aes128-gcm@openssh.com
|
||||
MACs hmac-sha2-512-etm@openssh.com,hmac-sha2-256-etm@openssh.com
|
||||
|
||||
# Authentication
|
||||
PermitRootLogin no
|
||||
PasswordAuthentication no
|
||||
PubkeyAuthentication yes
|
||||
AuthenticationMethods publickey
|
||||
MaxAuthTries 3
|
||||
AllowUsers deploy admin
|
||||
```
|
||||
MaxSessions 5
|
||||
LoginGraceTime 30
|
||||
|
||||
## Tunneling
|
||||
# Restrict users and groups
|
||||
AllowGroups ssh-users ops-team
|
||||
# AllowUsers deploy admin
|
||||
|
||||
# Disable unused authentication methods
|
||||
ChallengeResponseAuthentication no
|
||||
KerberosAuthentication no
|
||||
GSSAPIAuthentication no
|
||||
|
||||
# Forwarding controls
|
||||
AllowTcpForwarding yes
|
||||
AllowAgentForwarding no
|
||||
X11Forwarding no
|
||||
PermitTunnel no
|
||||
|
||||
# Security hardening
|
||||
ClientAliveInterval 300
|
||||
ClientAliveCountMax 2
|
||||
UsePAM yes
|
||||
UseDNS no
|
||||
PermitEmptyPasswords no
|
||||
PermitUserEnvironment no
|
||||
|
||||
# Logging
|
||||
SyslogFacility AUTH
|
||||
LogLevel VERBOSE
|
||||
|
||||
# SFTP subsystem
|
||||
Subsystem sftp /usr/lib/openssh/sftp-server -f AUTH -l INFO
|
||||
|
||||
# Match block: restrict deploy user to SFTP only
|
||||
Match User sftponly
|
||||
ForceCommand internal-sftp
|
||||
ChrootDirectory /home/%u
|
||||
AllowTcpForwarding no
|
||||
AllowAgentForwarding no
|
||||
X11Forwarding no
|
||||
```
|
||||
|
||||
```bash
|
||||
# Local port forward
|
||||
ssh -L 8080:internal:80 bastion
|
||||
# Validate configuration before restarting
|
||||
sshd -t
|
||||
|
||||
# Remote port forward
|
||||
ssh -R 8080:localhost:80 server
|
||||
# Restart sshd to apply changes
|
||||
systemctl restart sshd
|
||||
|
||||
# SOCKS proxy
|
||||
ssh -D 1080 server
|
||||
# Always keep an existing session open while testing
|
||||
# Open a NEW terminal to verify you can still connect before closing the old one
|
||||
```
|
||||
|
||||
## Best Practices
|
||||
## Bastion Host Setup
|
||||
|
||||
- Use ed25519 keys
|
||||
- Disable password auth
|
||||
- Use SSH agent forwarding carefully
|
||||
- Implement jump hosts/bastions
|
||||
```bash
|
||||
# On the bastion server, restrict forwarding to internal subnets only
|
||||
# /etc/ssh/sshd_config addition on bastion:
|
||||
AllowTcpForwarding yes
|
||||
PermitOpen 10.0.0.0/8:22 10.0.0.0/8:5432
|
||||
|
||||
# Disable shell access for jump-only users
|
||||
Match User jump-user
|
||||
PermitTTY no
|
||||
ForceCommand /usr/sbin/nologin
|
||||
AllowTcpForwarding yes
|
||||
|
||||
# Connect through the bastion from a client in one command
|
||||
ssh -J ops@bastion.example.com deploy@10.0.1.10
|
||||
|
||||
# Equivalent using ProxyCommand (older SSH versions)
|
||||
ssh -o ProxyCommand="ssh -W %h:%p ops@bastion.example.com" deploy@10.0.1.10
|
||||
|
||||
# Multi-hop: client -> bastion -> app-server -> db-server
|
||||
ssh -J ops@bastion,deploy@10.0.1.10 dba@10.0.2.50
|
||||
```
|
||||
|
||||
## SSH Tunneling
|
||||
|
||||
```bash
|
||||
# Local port forward: access remote service on localhost
|
||||
# Access remote PostgreSQL (10.0.2.50:5432) via bastion at localhost:5432
|
||||
ssh -L 5432:10.0.2.50:5432 ops@bastion.example.com -N
|
||||
|
||||
# Remote port forward: expose local service to the remote network
|
||||
# Make local dev server (localhost:3000) available on server port 8080
|
||||
ssh -R 8080:localhost:3000 user@server -N
|
||||
|
||||
# Dynamic SOCKS proxy: route all traffic through the server
|
||||
ssh -D 1080 user@server -N
|
||||
# Then configure browser or apps to use SOCKS5 proxy at localhost:1080
|
||||
|
||||
# Tunnel with a background process
|
||||
ssh -fN -L 5432:10.0.2.50:5432 ops@bastion.example.com
|
||||
# Find and kill the tunnel later
|
||||
ps aux | grep "ssh -fN" | grep -v grep
|
||||
kill <pid>
|
||||
|
||||
# Autossh for persistent tunnels (auto-reconnects)
|
||||
autossh -M 0 -f -N -L 5432:10.0.2.50:5432 ops@bastion.example.com \
|
||||
-o "ServerAliveInterval=30" -o "ServerAliveCountMax=3"
|
||||
```
|
||||
|
||||
## Agent Forwarding (Use with Caution)
|
||||
|
||||
```bash
|
||||
# Enable agent forwarding for a single connection
|
||||
ssh -A user@bastion
|
||||
|
||||
# From the bastion, your local keys are available to authenticate further
|
||||
ssh deploy@10.0.1.10 # Uses your local key via the agent
|
||||
|
||||
# SECURITY WARNING: Agent forwarding exposes your keys to anyone with root
|
||||
# on the intermediate host. Prefer ProxyJump instead.
|
||||
|
||||
# Safer alternative: ProxyJump does not expose the agent
|
||||
ssh -J ops@bastion deploy@10.0.1.10
|
||||
```
|
||||
|
||||
## SSH Key Restrictions in authorized_keys
|
||||
|
||||
```text
|
||||
# Restrict a key to a specific command only (backup key)
|
||||
command="/usr/local/bin/run-backup.sh",no-port-forwarding,no-X11-forwarding,no-agent-forwarding ssh-ed25519 AAAA... backup@example.com
|
||||
|
||||
# Restrict a key to specific source IPs
|
||||
from="10.0.0.0/24,192.168.1.0/24" ssh-ed25519 AAAA... admin@example.com
|
||||
|
||||
# Read-only SFTP key with chroot
|
||||
command="internal-sftp",no-port-forwarding,no-pty ssh-ed25519 AAAA... sftp-upload@example.com
|
||||
```
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
| Symptom | Diagnostic Command | Common Fix |
|
||||
|---|---|---|
|
||||
| Connection refused | `ss -tlnp \| grep 22` on server | Ensure sshd is running; check firewall rules |
|
||||
| Permission denied (publickey) | `ssh -vvv user@server` | Verify key is in authorized_keys, permissions 600/700 |
|
||||
| Host key verification failed | `ssh-keygen -R server` | Remove stale host key; verify server identity |
|
||||
| Connection timeout | `ssh -o ConnectTimeout=5 user@server` | Check network path, security groups, NACLs |
|
||||
| Slow SSH login | Check `UseDNS` in sshd_config | Set `UseDNS no`; check reverse DNS |
|
||||
| Broken pipe / dropped sessions | Add `ServerAliveInterval 60` to config | Configure keepalive on both client and server |
|
||||
| Agent forwarding not working | `ssh-add -l` on bastion | Ensure `-A` flag used and agent has keys loaded |
|
||||
| Tunnel port already in use | `ss -tlnp \| grep <port>` | Kill existing tunnel or use a different local port |
|
||||
|
||||
## Related Skills
|
||||
|
||||
- `linux-administration` -- General Linux system administration
|
||||
- `user-management` -- Managing the users who connect via SSH
|
||||
- `systemd-services` -- Managing sshd as a systemd service
|
||||
- `performance-tuning` -- Network tuning for SSH performance
|
||||
|
||||
@@ -9,68 +9,355 @@ metadata:
|
||||
|
||||
# Systemd Services
|
||||
|
||||
Manage system services with systemd.
|
||||
Create, manage, and monitor systemd services and timers. Covers unit file authoring, dependency management, socket activation, resource limits, journalctl log analysis, and production hardening.
|
||||
|
||||
## Service Unit File
|
||||
## When to Use
|
||||
|
||||
- Deploying an application as a managed background service
|
||||
- Replacing cron jobs with systemd timers for better logging and dependency control
|
||||
- Setting up socket activation for on-demand service startup
|
||||
- Configuring resource limits (CPU, memory, I/O) for services
|
||||
- Debugging service startup failures and runtime crashes
|
||||
- Managing service dependencies and ordering
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- Linux system running systemd (most modern distributions)
|
||||
- Root or sudo access for creating system-level unit files
|
||||
- Application binary or script to run as a service
|
||||
- Understanding of the application's start/stop lifecycle
|
||||
|
||||
## Service Unit File -- Complete Example
|
||||
|
||||
```ini
|
||||
# /etc/systemd/system/myapp.service
|
||||
[Unit]
|
||||
Description=My Application
|
||||
After=network.target
|
||||
Description=MyApp Production Server
|
||||
Documentation=https://docs.example.com/myapp
|
||||
After=network-online.target postgresql.service
|
||||
Wants=network-online.target
|
||||
Requires=postgresql.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
Type=notify
|
||||
User=myapp
|
||||
Group=myapp
|
||||
WorkingDirectory=/opt/myapp
|
||||
ExecStart=/opt/myapp/bin/start
|
||||
ExecStop=/opt/myapp/bin/stop
|
||||
Restart=always
|
||||
RestartSec=5
|
||||
|
||||
# Environment configuration
|
||||
EnvironmentFile=/etc/myapp/env
|
||||
Environment=NODE_ENV=production
|
||||
Environment=PORT=8080
|
||||
|
||||
# Execution
|
||||
ExecStartPre=/opt/myapp/bin/migrate --check
|
||||
ExecStart=/opt/myapp/bin/server --config /etc/myapp/config.yaml
|
||||
ExecStartPost=/opt/myapp/bin/healthcheck.sh
|
||||
ExecReload=/bin/kill -HUP $MAINPID
|
||||
ExecStop=/opt/myapp/bin/graceful-stop.sh
|
||||
|
||||
# Restart behavior
|
||||
Restart=on-failure
|
||||
RestartSec=5
|
||||
StartLimitIntervalSec=300
|
||||
StartLimitBurst=5
|
||||
|
||||
# Timeouts
|
||||
TimeoutStartSec=30
|
||||
TimeoutStopSec=30
|
||||
WatchdogSec=60
|
||||
|
||||
# Security hardening
|
||||
NoNewPrivileges=true
|
||||
ProtectSystem=strict
|
||||
ProtectHome=true
|
||||
PrivateTmp=true
|
||||
ReadWritePaths=/var/lib/myapp /var/log/myapp
|
||||
CapabilityBoundingSet=
|
||||
AmbientCapabilities=
|
||||
|
||||
# Logging
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
SyslogIdentifier=myapp
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
```
|
||||
|
||||
## Service Management
|
||||
## Service Management Commands
|
||||
|
||||
```bash
|
||||
# Reload systemd after creating or modifying unit files
|
||||
systemctl daemon-reload
|
||||
|
||||
# Start, stop, restart a service
|
||||
systemctl start myapp
|
||||
systemctl stop myapp
|
||||
systemctl restart myapp
|
||||
|
||||
# Reload service configuration without restart (if supported)
|
||||
systemctl reload myapp
|
||||
|
||||
# Enable service to start on boot
|
||||
systemctl enable myapp
|
||||
|
||||
# Enable and start in one command
|
||||
systemctl enable --now myapp
|
||||
|
||||
# Disable and stop
|
||||
systemctl disable --now myapp
|
||||
|
||||
# Check service status
|
||||
systemctl status myapp
|
||||
journalctl -u myapp -f
|
||||
|
||||
# Check if a service is active, enabled, or failed
|
||||
systemctl is-active myapp
|
||||
systemctl is-enabled myapp
|
||||
systemctl is-failed myapp
|
||||
|
||||
# List all running services
|
||||
systemctl list-units --type=service --state=running
|
||||
|
||||
# List all failed services
|
||||
systemctl list-units --type=service --state=failed
|
||||
|
||||
# Show all properties of a service
|
||||
systemctl show myapp
|
||||
|
||||
# Show specific property values
|
||||
systemctl show myapp -p MainPID,MemoryCurrent,CPUUsageNSec
|
||||
|
||||
# Mask a service (prevent it from being started at all)
|
||||
systemctl mask myapp
|
||||
|
||||
# Unmask
|
||||
systemctl unmask myapp
|
||||
|
||||
# Reset a failed service state
|
||||
systemctl reset-failed myapp
|
||||
```
|
||||
|
||||
## Timer (Cron Replacement)
|
||||
## Timer Units (Cron Replacement)
|
||||
|
||||
### Timer File
|
||||
|
||||
```ini
|
||||
# /etc/systemd/system/backup.timer
|
||||
[Unit]
|
||||
Description=Daily backup
|
||||
Description=Daily backup timer
|
||||
|
||||
[Timer]
|
||||
OnCalendar=daily
|
||||
# Run daily at 2:30 AM
|
||||
OnCalendar=*-*-* 02:30:00
|
||||
# If the system was off at the scheduled time, run when it boots
|
||||
Persistent=true
|
||||
# Add random delay up to 15 minutes to avoid thundering herd
|
||||
RandomizedDelaySec=900
|
||||
# Associate with a specific service (defaults to same name .service)
|
||||
Unit=backup.service
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
```
|
||||
|
||||
## Resource Limits
|
||||
### Corresponding Service File
|
||||
|
||||
```ini
|
||||
# /etc/systemd/system/backup.service
|
||||
[Unit]
|
||||
Description=Daily backup job
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
MemoryLimit=512M
|
||||
CPUQuota=50%
|
||||
Type=oneshot
|
||||
User=backup
|
||||
ExecStart=/usr/local/bin/run-backup.sh
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
```
|
||||
|
||||
## Best Practices
|
||||
### Timer Management
|
||||
|
||||
- Use Type=notify for better tracking
|
||||
- Implement proper restart policies
|
||||
- Use timers instead of cron
|
||||
- Set resource limits
|
||||
```bash
|
||||
# Common OnCalendar expressions:
|
||||
# minutely, hourly, daily, weekly, monthly
|
||||
# *-*-* 06:00:00 Daily at 6 AM
|
||||
# Mon..Fri *-*-* 09:00 Weekdays at 9 AM
|
||||
# *:0/15 Every 15 minutes
|
||||
|
||||
# Validate calendar expressions
|
||||
systemd-analyze calendar "Mon..Fri *-*-* 09:00"
|
||||
|
||||
# List all active timers
|
||||
systemctl list-timers --all
|
||||
|
||||
# Enable and start a timer
|
||||
systemctl enable --now backup.timer
|
||||
|
||||
# Run the associated service immediately (for testing)
|
||||
systemctl start backup.service
|
||||
```
|
||||
|
||||
## Socket Activation
|
||||
|
||||
```ini
|
||||
# /etc/systemd/system/myapp.socket
|
||||
[Unit]
|
||||
Description=MyApp Socket
|
||||
|
||||
[Socket]
|
||||
ListenStream=8080
|
||||
Accept=no
|
||||
# Optionally bind to a specific IP
|
||||
# ListenStream=10.0.1.10:8080
|
||||
|
||||
[Install]
|
||||
WantedBy=sockets.target
|
||||
```
|
||||
|
||||
```ini
|
||||
# /etc/systemd/system/myapp.service
|
||||
[Unit]
|
||||
Description=MyApp Server
|
||||
Requires=myapp.socket
|
||||
|
||||
[Service]
|
||||
Type=notify
|
||||
User=myapp
|
||||
ExecStart=/opt/myapp/bin/server
|
||||
# Service receives the socket file descriptor from systemd
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
```
|
||||
|
||||
```bash
|
||||
# Enable the socket (service starts on first connection)
|
||||
systemctl enable --now myapp.socket
|
||||
|
||||
# Check socket status
|
||||
systemctl status myapp.socket
|
||||
|
||||
# List all listening sockets
|
||||
systemctl list-sockets
|
||||
```
|
||||
|
||||
## Dependency Management
|
||||
|
||||
```bash
|
||||
# Key [Unit] directives for ordering and dependencies:
|
||||
# After= Start after these units (ordering only)
|
||||
# Requires= Hard dependency -- fail if this unit cannot start
|
||||
# Wants= Soft dependency -- try to start, don't fail if unavailable
|
||||
# PartOf= Stop this unit when the parent stops
|
||||
# Conflicts= Cannot run alongside this unit
|
||||
|
||||
# Visualize the dependency tree for a service
|
||||
systemctl list-dependencies myapp
|
||||
|
||||
# Show reverse dependencies (who depends on this unit)
|
||||
systemctl list-dependencies myapp --reverse
|
||||
|
||||
# Analyze boot order for a service
|
||||
systemd-analyze critical-chain myapp.service
|
||||
```
|
||||
|
||||
## Resource Limits (cgroups v2)
|
||||
|
||||
```ini
|
||||
# /etc/systemd/system/myapp.service.d/limits.conf
|
||||
# (drop-in override file)
|
||||
[Service]
|
||||
# Memory limits
|
||||
MemoryMax=1G
|
||||
MemoryHigh=768M
|
||||
|
||||
# CPU limits
|
||||
CPUQuota=200% # Up to 2 full CPU cores
|
||||
CPUWeight=100 # Relative weight (default=100)
|
||||
|
||||
# I/O limits
|
||||
IOWeight=50
|
||||
IOReadBandwidthMax=/dev/sda 100M
|
||||
IOWriteBandwidthMax=/dev/sda 50M
|
||||
|
||||
# Process limits
|
||||
LimitNOFILE=65535
|
||||
LimitNPROC=4096
|
||||
TasksMax=512
|
||||
|
||||
# Disable OOM killer (let the app handle it)
|
||||
OOMPolicy=continue
|
||||
```
|
||||
|
||||
```bash
|
||||
# Apply drop-in overrides without editing the main unit file
|
||||
mkdir -p /etc/systemd/system/myapp.service.d/
|
||||
|
||||
cat <<'EOF' > /etc/systemd/system/myapp.service.d/limits.conf
|
||||
[Service]
|
||||
MemoryMax=1G
|
||||
CPUQuota=200%
|
||||
EOF
|
||||
|
||||
systemctl daemon-reload
|
||||
systemctl restart myapp
|
||||
|
||||
# View current resource usage for a service
|
||||
systemctl status myapp # Shows Memory and CPU
|
||||
systemd-cgtop # Real-time cgroup resource usage
|
||||
|
||||
# Edit a service's overrides interactively
|
||||
systemctl edit myapp
|
||||
# This creates a drop-in file automatically
|
||||
```
|
||||
|
||||
## Journalctl Log Analysis
|
||||
|
||||
```bash
|
||||
# Follow logs for a service in real time
|
||||
journalctl -u myapp -f
|
||||
|
||||
# Show logs since last boot
|
||||
journalctl -u myapp -b
|
||||
|
||||
# Show logs for a specific time range
|
||||
journalctl -u myapp --since "2025-01-15 08:00" --until "2025-01-15 12:00"
|
||||
|
||||
# Show only error and above
|
||||
journalctl -u myapp -p err
|
||||
|
||||
# Show the last 100 lines with full messages (no truncation)
|
||||
journalctl -u myapp -n 100 --no-pager -l
|
||||
|
||||
# Show logs in JSON format (for parsing)
|
||||
journalctl -u myapp -o json-pretty --no-pager | head -50
|
||||
|
||||
# Check journal disk usage and vacuum old entries
|
||||
journalctl --disk-usage
|
||||
journalctl --rotate
|
||||
journalctl --vacuum-time=7d
|
||||
journalctl --vacuum-size=500M
|
||||
```
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
| Symptom | Diagnostic Command | Common Fix |
|
||||
|---|---|---|
|
||||
| Service fails to start | `systemctl status myapp`, `journalctl -u myapp -n 50` | Check ExecStart path, permissions, config syntax |
|
||||
| Service keeps restarting | `journalctl -u myapp --since "5 min ago"` | Check StartLimitBurst; look for crash in logs |
|
||||
| "Main process exited, code=exited, status=217" | `journalctl -u myapp` | User or group in unit file does not exist |
|
||||
| "Failed to set up mount namespacing" | Check ProtectSystem/PrivateTmp | Kernel too old or SELinux blocking; relax directives |
|
||||
| Timer not firing | `systemctl list-timers`, `systemctl status backup.timer` | Ensure timer is enabled; validate OnCalendar expression |
|
||||
| Service starts before dependency | Check After= and Requires= | Add `After=dependency.service` for ordering |
|
||||
| OOM killed | `journalctl -k \| grep oom`, `dmesg` | Increase MemoryMax or optimize application memory |
|
||||
| Cannot bind to port 80 | Check AmbientCapabilities | Add `CAP_NET_BIND_SERVICE` or use a higher port |
|
||||
|
||||
## Related Skills
|
||||
|
||||
- `linux-administration` -- General system administration context
|
||||
- `performance-tuning` -- Kernel tuning and resource optimization
|
||||
- `user-management` -- Service accounts and permissions
|
||||
- `backup-recovery` -- Scheduling backups with systemd timers
|
||||
|
||||
@@ -9,61 +9,359 @@ metadata:
|
||||
|
||||
# User Management
|
||||
|
||||
Manage users, groups, and permissions.
|
||||
Manage users, groups, permissions, sudo access, PAM modules, and LDAP integration on Linux systems. Includes practical scripts for bulk user operations and access auditing.
|
||||
|
||||
## When to Use
|
||||
|
||||
- Creating and managing local user accounts on Linux servers
|
||||
- Configuring sudo access with fine-grained privilege controls
|
||||
- Setting up group-based access control for teams
|
||||
- Integrating Linux hosts with LDAP or Active Directory for centralized auth
|
||||
- Auditing user accounts, permissions, and access patterns
|
||||
- Automating bulk user provisioning and deprovisioning
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- Root or sudo access on the target system
|
||||
- `shadow-utils` package (provides useradd, usermod, etc.) -- installed by default
|
||||
- `libpam-modules` for PAM configuration
|
||||
- For LDAP: `sssd`, `realmd`, `libpam-ldapd`, or `nslcd` packages
|
||||
- For auditing: `auditd` package
|
||||
|
||||
## User Operations
|
||||
|
||||
### Creating Users
|
||||
|
||||
```bash
|
||||
# Create user
|
||||
useradd -m -s /bin/bash username
|
||||
passwd username
|
||||
# Create a user with home directory, default shell, and comment
|
||||
useradd -m -s /bin/bash -c "Jane Smith" jsmith
|
||||
|
||||
# Delete user
|
||||
userdel -r username
|
||||
# Set the user's password interactively
|
||||
passwd jsmith
|
||||
|
||||
# Modify user
|
||||
usermod -aG sudo username
|
||||
usermod -s /bin/zsh username
|
||||
# Create a user with a specific UID and primary group
|
||||
useradd -m -s /bin/bash -u 1500 -g developers -c "Deploy Account" deploy
|
||||
|
||||
# Create a system account (no home, no login shell) for running services
|
||||
useradd -r -s /usr/sbin/nologin -d /opt/myapp -c "MyApp Service Account" myapp
|
||||
|
||||
# Create a user with an expiration date (contractor access)
|
||||
useradd -m -s /bin/bash -e 2025-12-31 -c "Contractor - Bob Lee" blee
|
||||
|
||||
# Create user and add to multiple supplementary groups at creation time
|
||||
useradd -m -s /bin/bash -G docker,developers,ssh-users -c "Dev User" devuser
|
||||
```
|
||||
|
||||
### Modifying Users
|
||||
|
||||
```bash
|
||||
# Add a user to a supplementary group (preserving existing groups with -a)
|
||||
usermod -aG sudo jsmith
|
||||
usermod -aG docker,developers jsmith
|
||||
|
||||
# Change the user's login shell
|
||||
usermod -s /bin/zsh jsmith
|
||||
|
||||
# Change the user's home directory and move existing files
|
||||
usermod -d /home/jsmith-new -m jsmith
|
||||
|
||||
# Lock a user account (disable login without deleting)
|
||||
usermod -L jsmith
|
||||
|
||||
# Unlock a user account
|
||||
usermod -U jsmith
|
||||
|
||||
# Set an account expiration date
|
||||
usermod -e 2025-06-30 blee
|
||||
|
||||
# Change a user's login name
|
||||
usermod -l jsmith-new jsmith
|
||||
|
||||
# Force password change on next login
|
||||
chage -d 0 jsmith
|
||||
|
||||
# Set password aging: min 7 days, max 90 days, warn 14 days before
|
||||
chage -m 7 -M 90 -W 14 jsmith
|
||||
|
||||
# View password aging info
|
||||
chage -l jsmith
|
||||
```
|
||||
|
||||
### Deleting Users
|
||||
|
||||
```bash
|
||||
# Remove a user and their home directory
|
||||
userdel -r jsmith
|
||||
|
||||
# Remove a user but keep their home directory (for auditing)
|
||||
userdel jsmith
|
||||
|
||||
# Find and reassign files owned by a deleted user (by UID)
|
||||
find / -uid 1500 -exec chown newowner:newgroup {} \;
|
||||
```
|
||||
|
||||
## Group Management
|
||||
|
||||
```bash
|
||||
# Create group
|
||||
# Create a new group
|
||||
groupadd developers
|
||||
|
||||
# Add user to group
|
||||
usermod -aG developers username
|
||||
gpasswd -a username developers
|
||||
# Create a group with a specific GID
|
||||
groupadd -g 2000 devops
|
||||
|
||||
# Remove from group
|
||||
gpasswd -d username developers
|
||||
# Add a user to a group
|
||||
usermod -aG developers jsmith
|
||||
# Alternative using gpasswd
|
||||
gpasswd -a jsmith developers
|
||||
|
||||
# Remove a user from a group
|
||||
gpasswd -d jsmith developers
|
||||
|
||||
# Set group administrators (can add/remove members without root)
|
||||
gpasswd -A jsmith developers
|
||||
|
||||
# Delete a group
|
||||
groupdel developers
|
||||
|
||||
# List all groups a user belongs to
|
||||
groups jsmith
|
||||
id jsmith
|
||||
|
||||
# List all members of a group
|
||||
getent group developers
|
||||
|
||||
# Show all groups on the system
|
||||
cat /etc/group | cut -d: -f1 | sort
|
||||
```
|
||||
|
||||
## Sudo Configuration
|
||||
|
||||
```bash
|
||||
# /etc/sudoers.d/developers
|
||||
%developers ALL=(ALL) NOPASSWD: /usr/bin/docker
|
||||
username ALL=(ALL) NOPASSWD: ALL
|
||||
# Always edit sudoers via visudo (syntax validation prevents lockout)
|
||||
visudo
|
||||
|
||||
# Better: use drop-in files in /etc/sudoers.d/
|
||||
visudo -f /etc/sudoers.d/developers
|
||||
```
|
||||
|
||||
## File Permissions
|
||||
### /etc/sudoers.d/developers
|
||||
|
||||
```text
|
||||
# Allow the developers group to restart specific services
|
||||
%developers ALL=(ALL) NOPASSWD: /usr/bin/systemctl restart myapp, /usr/bin/systemctl status myapp
|
||||
|
||||
# Allow a deploy user full sudo with no password
|
||||
deploy ALL=(ALL) NOPASSWD: ALL
|
||||
|
||||
# Allow ops team to run docker commands only
|
||||
%ops ALL=(ALL) NOPASSWD: /usr/bin/docker, /usr/bin/docker-compose
|
||||
|
||||
# Allow a user to run commands as a specific service account
|
||||
jsmith ALL=(myapp) NOPASSWD: /opt/myapp/bin/*
|
||||
|
||||
# Restrict to specific hosts (useful with centralized sudoers)
|
||||
jsmith dbservers=(root) /usr/bin/systemctl restart postgresql
|
||||
|
||||
# Log all sudo commands to a dedicated file
|
||||
Defaults log_output
|
||||
Defaults!/usr/bin/sudoreplay !log_output
|
||||
Defaults logfile="/var/log/sudo.log"
|
||||
|
||||
# Require password re-entry every 5 minutes (default is 15)
|
||||
Defaults timestamp_timeout=5
|
||||
|
||||
# Require password for sudo even if user has NOPASSWD elsewhere
|
||||
Defaults:jsmith !authenticate
|
||||
```
|
||||
|
||||
```bash
|
||||
chmod 755 file # rwxr-xr-x
|
||||
chmod u+x file # Add execute for user
|
||||
chown user:group file # Change ownership
|
||||
chown -R user:group dir/
|
||||
# Validate sudoers syntax without applying
|
||||
visudo -c
|
||||
|
||||
# ACLs
|
||||
setfacl -m u:user:rx file
|
||||
getfacl file
|
||||
# Check what sudo permissions a user has
|
||||
sudo -l -U jsmith
|
||||
|
||||
# Test a specific sudo command as a user
|
||||
sudo -u myapp /opt/myapp/bin/healthcheck.sh
|
||||
```
|
||||
|
||||
## Best Practices
|
||||
## File Permissions and ACLs
|
||||
|
||||
- Use groups for access control
|
||||
- Minimal sudo privileges
|
||||
- Regular access reviews
|
||||
- Strong password policies
|
||||
```bash
|
||||
# Standard permissions
|
||||
chmod 755 /opt/myapp # rwxr-xr-x
|
||||
chmod 640 /etc/myapp.conf # rw-r-----
|
||||
chmod u+x script.sh # Add execute for owner
|
||||
chmod g+w shared-dir/ # Add write for group
|
||||
chmod o-rwx private-file # Remove all permissions for others
|
||||
|
||||
# Change ownership
|
||||
chown deploy:developers /opt/myapp
|
||||
chown -R deploy:developers /opt/myapp/ # Recursive
|
||||
|
||||
# Set the SGID bit (new files inherit group ownership)
|
||||
chmod g+s /opt/shared/
|
||||
|
||||
# Set the sticky bit (only owner can delete their files)
|
||||
chmod +t /tmp/shared/
|
||||
|
||||
# Access Control Lists (ACLs) for fine-grained control
|
||||
# Grant read-execute to a specific user on a directory
|
||||
setfacl -m u:jsmith:rx /opt/myapp/logs/
|
||||
|
||||
# Grant read-write to a group
|
||||
setfacl -m g:developers:rw /opt/shared/
|
||||
|
||||
# Set default ACL (applied to new files created in the directory)
|
||||
setfacl -d -m g:developers:rw /opt/shared/
|
||||
|
||||
# View ACLs
|
||||
getfacl /opt/shared/
|
||||
|
||||
# Remove a specific ACL entry
|
||||
setfacl -x u:jsmith /opt/myapp/logs/
|
||||
|
||||
# Remove all ACLs
|
||||
setfacl -b /opt/shared/
|
||||
```
|
||||
|
||||
## PAM Configuration
|
||||
|
||||
```bash
|
||||
# PAM config files are in /etc/pam.d/
|
||||
# Each file controls auth for a specific service (sshd, login, sudo, etc.)
|
||||
|
||||
# Enforce password complexity via pam_pwquality
|
||||
# /etc/pam.d/common-password (Debian) or /etc/pam.d/system-auth (RHEL)
|
||||
password requisite pam_pwquality.so retry=3 minlen=12 dcredit=-1 ucredit=-1 ocredit=-1 lcredit=-1
|
||||
|
||||
# Configure /etc/security/pwquality.conf
|
||||
minlen = 12
|
||||
dcredit = -1
|
||||
ucredit = -1
|
||||
ocredit = -1
|
||||
lcredit = -1
|
||||
maxrepeat = 3
|
||||
dictcheck = 1
|
||||
|
||||
# Limit concurrent logins per user
|
||||
# /etc/security/limits.conf
|
||||
jsmith hard maxlogins 3
|
||||
@developers hard maxlogins 5
|
||||
|
||||
# Lock account after 5 failed login attempts
|
||||
# /etc/pam.d/common-auth (Debian)
|
||||
auth required pam_faillock.so preauth silent deny=5 unlock_time=900
|
||||
auth required pam_faillock.so authfail deny=5 unlock_time=900
|
||||
|
||||
# View failed login attempts
|
||||
faillock --user jsmith
|
||||
|
||||
# Unlock a locked account
|
||||
faillock --user jsmith --reset
|
||||
```
|
||||
|
||||
## LDAP / Active Directory Integration
|
||||
|
||||
```bash
|
||||
# Install SSSD and realmd for AD integration (Ubuntu/Debian)
|
||||
apt install -y sssd realmd adcli sssd-tools libnss-sss libpam-sss
|
||||
|
||||
# Install SSSD and realmd (RHEL/CentOS)
|
||||
dnf install -y sssd realmd adcli sssd-tools oddjob oddjob-mkhomedir
|
||||
|
||||
# Discover and join an Active Directory domain
|
||||
realm discover corp.example.com
|
||||
realm join corp.example.com -U admin@CORP.EXAMPLE.COM
|
||||
|
||||
# Verify the join
|
||||
realm list
|
||||
|
||||
# Allow specific AD groups to log in
|
||||
realm permit -g "Linux Admins@corp.example.com"
|
||||
realm permit -g "Developers@corp.example.com"
|
||||
|
||||
# Deny all except permitted groups
|
||||
realm deny --all
|
||||
realm permit -g "Linux Admins@corp.example.com"
|
||||
|
||||
# Restart SSSD after config changes
|
||||
systemctl restart sssd
|
||||
|
||||
# Test LDAP user lookup
|
||||
id jsmith
|
||||
getent passwd jsmith
|
||||
|
||||
# Grant sudo to an AD group
|
||||
echo '%linux\ admins ALL=(ALL) ALL' > /etc/sudoers.d/ad-admins
|
||||
```
|
||||
|
||||
## Bulk User Management Scripts
|
||||
|
||||
### Bulk User Creation from CSV
|
||||
|
||||
```bash
|
||||
#!/bin/bash
|
||||
# bulk-create-users.sh
|
||||
# CSV format: username,fullname,groups,shell
|
||||
# Example: jsmith,Jane Smith,developers;docker,/bin/bash
|
||||
|
||||
CSV_FILE="${1:?Usage: $0 <users.csv>}"
|
||||
|
||||
while IFS=',' read -r username fullname groups shell; do
|
||||
# Skip header line
|
||||
[[ "$username" == "username" ]] && continue
|
||||
|
||||
if id "$username" &>/dev/null; then
|
||||
echo "SKIP: User $username already exists"
|
||||
continue
|
||||
fi
|
||||
|
||||
# Replace semicolons with commas for -G flag
|
||||
group_list="${groups//;/,}"
|
||||
|
||||
useradd -m -s "$shell" -c "$fullname" -G "$group_list" "$username"
|
||||
# Generate a random temporary password
|
||||
temp_pass=$(openssl rand -base64 12)
|
||||
echo "$username:$temp_pass" | chpasswd
|
||||
chage -d 0 "$username" # Force password change at first login
|
||||
|
||||
echo "CREATED: $username (groups: $group_list) temp-pass: $temp_pass"
|
||||
done < "$CSV_FILE"
|
||||
```
|
||||
|
||||
### Quick Access Audit Commands
|
||||
|
||||
```bash
|
||||
# List non-system users (UID >= 1000)
|
||||
awk -F: '$3 >= 1000 && $3 < 65534 { printf "%-20s UID=%-6s Shell=%s\n", $1, $3, $7 }' /etc/passwd
|
||||
|
||||
# List users with sudo access
|
||||
getent group sudo wheel 2>/dev/null
|
||||
|
||||
# Find accounts that have never logged in
|
||||
lastlog | awk '$0 ~ /Never logged in/ { print $1 }'
|
||||
|
||||
# Find accounts with empty passwords
|
||||
awk -F: '($2 == "" || $2 == "!") { print $1 }' /etc/shadow 2>/dev/null
|
||||
```
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
| Symptom | Diagnostic Command | Common Fix |
|
||||
|---|---|---|
|
||||
| User cannot log in | `passwd -S username`, `faillock --user username` | Unlock account, reset password, check shell |
|
||||
| "not in sudoers" error | `sudo -l -U username` | Add user to sudo group or create sudoers.d file |
|
||||
| Group membership not applied | `id username`, `groups username` | User must log out and back in for new groups |
|
||||
| LDAP/AD user not found | `id aduser`, `sssctl user-show aduser` | Check SSSD status, clear cache: `sss_cache -E` |
|
||||
| Permission denied on file | `ls -la file`, `getfacl file` | Fix ownership/permissions, check SELinux context |
|
||||
| PAM lockout after failed attempts | `faillock --user username` | `faillock --user username --reset` |
|
||||
| Home directory not created | Check `/etc/login.defs` CREATEHOME | Use `useradd -m` or enable `pam_mkhomedir` |
|
||||
| Password policy not enforced | Check `/etc/pam.d/common-password` | Install and configure `pam_pwquality` |
|
||||
|
||||
## Related Skills
|
||||
|
||||
- `linux-administration` -- General Linux server management
|
||||
- `ssh-configuration` -- SSH key-based authentication for managed users
|
||||
- `systemd-services` -- Service accounts and systemd user instances
|
||||
- `performance-tuning` -- Resource limits per user via cgroups and ulimits
|
||||
|
||||
@@ -9,48 +9,294 @@ metadata:
|
||||
|
||||
# Windows Server Administration
|
||||
|
||||
Windows Server management and PowerShell automation.
|
||||
Windows Server management and PowerShell automation for production workloads including IIS web hosting, Active Directory domain services, and system maintenance.
|
||||
|
||||
## Server Roles
|
||||
## When to Use
|
||||
|
||||
- Provisioning or configuring Windows Server 2019/2022 instances
|
||||
- Setting up IIS websites, application pools, and bindings
|
||||
- Managing Active Directory users, groups, and Group Policy
|
||||
- Automating administrative tasks with PowerShell
|
||||
- Reviewing Windows Event Logs for troubleshooting
|
||||
- Applying and managing Windows Updates on servers
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- Administrator account on the target server
|
||||
- PowerShell 5.1+ (built-in) or PowerShell 7+ installed
|
||||
- Remote Desktop or WinRM access configured
|
||||
- Windows Server 2019 or 2022 (Desktop Experience or Server Core)
|
||||
|
||||
## PowerShell Administration Essentials
|
||||
|
||||
```powershell
|
||||
# Install IIS
|
||||
Install-WindowsFeature -Name Web-Server -IncludeManagementTools
|
||||
# Check PowerShell version
|
||||
$PSVersionTable.PSVersion
|
||||
|
||||
# Install AD DS
|
||||
# Get system information
|
||||
Get-ComputerInfo | Select-Object CsName, OsName, OsVersion, OsArchitecture
|
||||
|
||||
# List running processes sorted by CPU
|
||||
Get-Process | Sort-Object CPU -Descending | Select-Object -First 20
|
||||
|
||||
# List all services and their status
|
||||
Get-Service | Where-Object { $_.Status -eq 'Running' }
|
||||
|
||||
# Restart a service
|
||||
Restart-Service -Name W3SVC -Force
|
||||
|
||||
# Get disk space on all drives
|
||||
Get-PSDrive -PSProvider FileSystem | Select-Object Name, @{N='Used(GB)';E={[math]::Round($_.Used/1GB,2)}}, @{N='Free(GB)';E={[math]::Round($_.Free/1GB,2)}}
|
||||
|
||||
# Check uptime
|
||||
(Get-Date) - (Get-CimInstance Win32_OperatingSystem).LastBootUpTime
|
||||
|
||||
# Open firewall port
|
||||
New-NetFirewallRule -DisplayName "Allow HTTPS" -Direction Inbound -Protocol TCP -LocalPort 443 -Action Allow
|
||||
|
||||
# List firewall rules
|
||||
Get-NetFirewallRule | Where-Object { $_.Enabled -eq 'True' -and $_.Direction -eq 'Inbound' } | Select-Object DisplayName, Action
|
||||
|
||||
# Set DNS client server addresses
|
||||
Set-DnsClientServerAddress -InterfaceAlias "Ethernet" -ServerAddresses ("10.0.0.2","10.0.0.3")
|
||||
|
||||
# PowerShell remoting to another server
|
||||
Enter-PSSession -ComputerName server02 -Credential (Get-Credential)
|
||||
|
||||
# Run a command on multiple remote servers
|
||||
Invoke-Command -ComputerName server01,server02,server03 -ScriptBlock { Get-Service W3SVC }
|
||||
```
|
||||
|
||||
## Server Roles and Features
|
||||
|
||||
```powershell
|
||||
# List all available roles and features
|
||||
Get-WindowsFeature
|
||||
|
||||
# Install IIS with management tools
|
||||
Install-WindowsFeature -Name Web-Server -IncludeManagementTools -IncludeAllSubFeature
|
||||
|
||||
# Install Active Directory Domain Services
|
||||
Install-WindowsFeature -Name AD-Domain-Services -IncludeManagementTools
|
||||
|
||||
# List installed features
|
||||
Get-WindowsFeature | Where-Object Installed
|
||||
# Install DNS Server
|
||||
Install-WindowsFeature -Name DNS -IncludeManagementTools
|
||||
|
||||
# Install DHCP Server
|
||||
Install-WindowsFeature -Name DHCP -IncludeManagementTools
|
||||
|
||||
# Install File Server with deduplication
|
||||
Install-WindowsFeature -Name FS-FileServer, FS-Data-Deduplication
|
||||
|
||||
# List installed features only
|
||||
Get-WindowsFeature | Where-Object Installed | Select-Object Name, InstallState
|
||||
|
||||
# Remove a feature
|
||||
Uninstall-WindowsFeature -Name Telnet-Client
|
||||
```
|
||||
|
||||
## System Information
|
||||
## IIS Web Server Setup
|
||||
|
||||
```powershell
|
||||
Get-ComputerInfo
|
||||
Get-Process
|
||||
Get-Service
|
||||
Get-EventLog -LogName System -Newest 50
|
||||
```
|
||||
# Import the IIS administration module
|
||||
Import-Module WebAdministration
|
||||
|
||||
## IIS Management
|
||||
# Create a new application pool
|
||||
New-WebAppPool -Name "ProductionPool"
|
||||
Set-ItemProperty IIS:\AppPools\ProductionPool -Name processModel.identityType -Value 3 # NetworkService
|
||||
Set-ItemProperty IIS:\AppPools\ProductionPool -Name managedRuntimeVersion -Value "" # No managed code (reverse proxy)
|
||||
|
||||
```powershell
|
||||
# Create website
|
||||
New-Website -Name "MyApp" -Port 80 -PhysicalPath "C:\inetpub\myapp"
|
||||
# Create a new website
|
||||
New-Website -Name "MyApp" `
|
||||
-Port 443 `
|
||||
-Protocol https `
|
||||
-PhysicalPath "C:\inetpub\myapp" `
|
||||
-ApplicationPool "ProductionPool" `
|
||||
-SslFlags 1
|
||||
|
||||
# Create app pool
|
||||
New-WebAppPool -Name "MyAppPool"
|
||||
# Add an HTTP binding that redirects to HTTPS
|
||||
New-WebBinding -Name "MyApp" -Protocol http -Port 80
|
||||
|
||||
# Start/Stop
|
||||
# Bind an SSL certificate to the HTTPS site
|
||||
$cert = Get-ChildItem -Path Cert:\LocalMachine\My | Where-Object { $_.Subject -like "*example.com*" }
|
||||
New-Item IIS:\SslBindings\0.0.0.0!443 -Value $cert
|
||||
|
||||
# Create a virtual directory
|
||||
New-WebVirtualDirectory -Site "MyApp" -Name "static" -PhysicalPath "C:\inetpub\static"
|
||||
|
||||
# Start, stop, and restart a site
|
||||
Start-Website -Name "MyApp"
|
||||
Stop-Website -Name "MyApp"
|
||||
Stop-Website -Name "MyApp"
|
||||
Restart-WebAppPool -Name "ProductionPool"
|
||||
|
||||
# List all websites and their state
|
||||
Get-Website | Select-Object Name, State, PhysicalPath, @{N='Bindings';E={$_.Bindings.Collection.bindingInformation}}
|
||||
|
||||
# Enable IIS logging with W3C format
|
||||
Set-WebConfigurationProperty -PSPath "IIS:\Sites\MyApp" `
|
||||
-Filter "system.webServer/httpLogging" `
|
||||
-Name "dontLog" -Value $false
|
||||
|
||||
# URL Rewrite: redirect HTTP to HTTPS (requires URL Rewrite module)
|
||||
# web.config rule:
|
||||
@'
|
||||
<rule name="HTTP to HTTPS" stopProcessing="true">
|
||||
<match url="(.*)" />
|
||||
<conditions>
|
||||
<add input="{HTTPS}" pattern="off" />
|
||||
</conditions>
|
||||
<action type="Redirect" url="https://{HTTP_HOST}/{R:1}" redirectType="Permanent" />
|
||||
</rule>
|
||||
'@
|
||||
```
|
||||
|
||||
## Best Practices
|
||||
## Active Directory Basics
|
||||
|
||||
- Use Server Core when possible
|
||||
- Implement Windows Admin Center
|
||||
- Regular Windows Update
|
||||
- PowerShell remoting over WinRM
|
||||
- Active Directory best practices
|
||||
```powershell
|
||||
# Promote server to a new domain controller in a new forest
|
||||
Install-ADDSForest `
|
||||
-DomainName "corp.example.com" `
|
||||
-DomainNetBIOSName "CORP" `
|
||||
-InstallDns:$true `
|
||||
-SafeModeAdministratorPassword (ConvertTo-SecureString "P@ssw0rd!" -AsPlainText -Force) `
|
||||
-Force:$true
|
||||
|
||||
# Create an Organizational Unit
|
||||
New-ADOrganizationalUnit -Name "Engineering" -Path "DC=corp,DC=example,DC=com"
|
||||
|
||||
# Create a new AD user
|
||||
New-ADUser -Name "Jane Smith" `
|
||||
-SamAccountName "jsmith" `
|
||||
-UserPrincipalName "jsmith@corp.example.com" `
|
||||
-Path "OU=Engineering,DC=corp,DC=example,DC=com" `
|
||||
-AccountPassword (ConvertTo-SecureString "TempP@ss1" -AsPlainText -Force) `
|
||||
-Enabled $true `
|
||||
-ChangePasswordAtLogon $true
|
||||
|
||||
# Add user to a group
|
||||
Add-ADGroupMember -Identity "Domain Admins" -Members "jsmith"
|
||||
|
||||
# Search for users in an OU
|
||||
Get-ADUser -Filter * -SearchBase "OU=Engineering,DC=corp,DC=example,DC=com" | Select-Object Name, SamAccountName, Enabled
|
||||
|
||||
# Disable a user account
|
||||
Disable-ADAccount -Identity "jsmith"
|
||||
|
||||
# Unlock a locked-out account
|
||||
Unlock-ADAccount -Identity "jsmith"
|
||||
|
||||
# Reset a user password
|
||||
Set-ADAccountPassword -Identity "jsmith" -Reset -NewPassword (ConvertTo-SecureString "NewP@ss1" -AsPlainText -Force)
|
||||
|
||||
# List all domain controllers
|
||||
Get-ADDomainController -Filter * | Select-Object Name, IPv4Address, Site
|
||||
|
||||
# Check AD replication status
|
||||
Get-ADReplicationPartnerMetadata -Target "dc01.corp.example.com"
|
||||
repadmin /replsummary
|
||||
```
|
||||
|
||||
## Windows Update Management
|
||||
|
||||
```powershell
|
||||
# Install the PSWindowsUpdate module (from PowerShell Gallery)
|
||||
Install-Module -Name PSWindowsUpdate -Force
|
||||
|
||||
# Check for available updates
|
||||
Get-WindowsUpdate
|
||||
|
||||
# Install all available updates (auto-reboot if needed)
|
||||
Install-WindowsUpdate -AcceptAll -AutoReboot
|
||||
|
||||
# Install only critical and security updates
|
||||
Install-WindowsUpdate -Category "Security Updates","Critical Updates" -AcceptAll
|
||||
|
||||
# View update history
|
||||
Get-WUHistory | Select-Object -First 20 Title, Date, Result
|
||||
|
||||
# Schedule monthly patching via Task Scheduler
|
||||
$action = New-ScheduledTaskAction -Execute "powershell.exe" `
|
||||
-Argument "-NoProfile -Command Install-WindowsUpdate -AcceptAll -AutoReboot"
|
||||
$trigger = New-ScheduledTaskTrigger -Weekly -DaysOfWeek Sunday -At 2am
|
||||
Register-ScheduledTask -TaskName "MonthlyPatching" -Action $action -Trigger $trigger -User "SYSTEM" -RunLevel Highest
|
||||
|
||||
# WSUS configuration via Group Policy (registry keys)
|
||||
# HKLM:\SOFTWARE\Policies\Microsoft\Windows\WindowsUpdate
|
||||
# WUServer = http://wsus.corp.example.com:8530
|
||||
# WUStatusServer = http://wsus.corp.example.com:8530
|
||||
```
|
||||
|
||||
## Event Log Analysis
|
||||
|
||||
```powershell
|
||||
# View the 50 most recent System log errors
|
||||
Get-EventLog -LogName System -EntryType Error -Newest 50
|
||||
|
||||
# Search for specific event IDs (e.g., unexpected shutdowns = 6008)
|
||||
Get-EventLog -LogName System -InstanceId 6008
|
||||
|
||||
# Use Get-WinEvent for advanced filtering (newer cmdlet)
|
||||
Get-WinEvent -FilterHashtable @{
|
||||
LogName = 'Application'
|
||||
Level = 2 # Error
|
||||
StartTime = (Get-Date).AddDays(-1)
|
||||
} | Select-Object TimeCreated, Id, Message -First 20
|
||||
|
||||
# Search Security log for failed logons (Event ID 4625)
|
||||
Get-WinEvent -FilterHashtable @{
|
||||
LogName = 'Security'
|
||||
Id = 4625
|
||||
} | Select-Object TimeCreated, @{N='Account';E={$_.Properties[5].Value}}, @{N='Source';E={$_.Properties[19].Value}} -First 30
|
||||
|
||||
# Export events to CSV for analysis
|
||||
Get-WinEvent -FilterHashtable @{ LogName='System'; Level=1,2 } |
|
||||
Export-Csv -Path C:\Logs\system-errors.csv -NoTypeInformation
|
||||
|
||||
# Clear old event log entries (use cautiously)
|
||||
Clear-EventLog -LogName Application
|
||||
|
||||
# Set maximum log size
|
||||
Limit-EventLog -LogName Application -MaximumSize 512MB -OverflowAction OverwriteAsNeeded
|
||||
```
|
||||
|
||||
## Scheduled Tasks
|
||||
|
||||
```powershell
|
||||
# Create a scheduled task to run a script daily at 3 AM
|
||||
$action = New-ScheduledTaskAction -Execute "powershell.exe" `
|
||||
-Argument "-NoProfile -File C:\Scripts\daily-maintenance.ps1"
|
||||
$trigger = New-ScheduledTaskTrigger -Daily -At 3am
|
||||
$settings = New-ScheduledTaskSettingsSet -StartWhenAvailable -DontStopOnIdleEnd
|
||||
Register-ScheduledTask -TaskName "DailyMaintenance" -Action $action -Trigger $trigger -Settings $settings -User "SYSTEM"
|
||||
|
||||
# List all scheduled tasks
|
||||
Get-ScheduledTask | Where-Object { $_.State -ne 'Disabled' } | Select-Object TaskName, State, TaskPath
|
||||
|
||||
# Run a task immediately
|
||||
Start-ScheduledTask -TaskName "DailyMaintenance"
|
||||
|
||||
# Disable and remove a task
|
||||
Disable-ScheduledTask -TaskName "DailyMaintenance"
|
||||
Unregister-ScheduledTask -TaskName "DailyMaintenance" -Confirm:$false
|
||||
```
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
| Symptom | Diagnostic Command | Common Fix |
|
||||
|---|---|---|
|
||||
| IIS site returns 503 | `Get-WebAppPoolState` | Restart the application pool; check Event Log for crash |
|
||||
| High CPU on server | `Get-Process \| Sort CPU -Desc` | Identify process; check for runaway w3wp or service |
|
||||
| Disk running low | `Get-PSDrive -PSProvider FileSystem` | Clear temp files, IIS logs, Windows Update cache |
|
||||
| AD account locked out | `Search-ADAccount -LockedOut` | `Unlock-ADAccount`; find lockout source in Security log |
|
||||
| Windows Update fails | `Get-WindowsUpdate -Verbose` | Run `sfc /scannow`, reset update components |
|
||||
| Service fails to start | `Get-EventLog -LogName System -Newest 20` | Check dependencies, credentials, and port conflicts |
|
||||
| RDP connection refused | `Get-ItemProperty 'HKLM:\System\...\Terminal Server'` | Ensure RDP is enabled and firewall allows port 3389 |
|
||||
| DNS resolution fails | `Resolve-DnsName example.com` | Check DNS server settings and forwarder config |
|
||||
|
||||
## Related Skills
|
||||
|
||||
- `linux-administration` -- Cross-platform comparison and hybrid management
|
||||
- `ssh-configuration` -- SSH access for Windows OpenSSH Server
|
||||
- `user-management` -- Parallel concepts for Linux user/group management
|
||||
- `systemd-services` -- Linux equivalent of Windows Services and Task Scheduler
|
||||
- `performance-tuning` -- Performance monitoring and optimization patterns
|
||||
|
||||
Reference in New Issue
Block a user