# === 下载安装 ===
cd /opt
wget https://github.com/prometheus/prometheus/releases/download/v2.52.0/prometheus-2.52.0.linux-amd64.tar.gz
tar xzf prometheus-2.52.0.linux-amd64.tar.gz
ln -s prometheus-2.52.0.linux-amd64 prometheus

# === 创建 systemd 服务 ===
cat > /etc/systemd/system/prometheus.service << 'EOF'
[Unit]
Description=Prometheus
After=network.target

[Service]
User=prometheus
Group=prometheus
Type=simple
ExecStart=/opt/prometheus/prometheus \
  --config.file=/opt/prometheus/prometheus.yml \
  --storage.tsdb.path=/data/prometheus \
  --storage.tsdb.retention.time=30d \
  --web.enable-lifecycle \
  --web.external-url=http://prometheus.example.com
ExecReload=/bin/kill -HUP $MAINPID
Restart=on-failure

[Install]
WantedBy=multi-user.target
EOF

useradd -r -s /sbin/nologin prometheus
mkdir -p /data/prometheus
chown -R prometheus:prometheus /opt/prometheus /data/prometheus
systemctl daemon-reload && systemctl enable --now prometheus

global:
  scrape_interval: 15s
  evaluation_interval: 15s
  external_labels:
    datacenter: 'bj-idc-01'
    env: 'production'

# 告警规则文件
rule_files:
  - 'rules/*.yml'

# 告警管理器
alerting:
  alertmanagers:
    - static_configs:
        - targets: ['localhost:9093']

# 采集目标
scrape_configs:
  # Prometheus 自身
  - job_name: 'prometheus'
    static_configs:
      - targets: ['localhost:9090']

  # Node Exporter (系统指标)
  - job_name: 'node'
    scrape_interval: 30s
    static_configs:
      - targets:
          - '192.168.1.11:9100'
          - '192.168.1.12:9100'
          - '192.168.1.13:9100'
        labels:
          env: 'production'
    # 基于文件的动态发现
    file_sd_configs:
      - files:
          - '/opt/prometheus/targets/node/*.json'
        refresh_interval: 5m

  # MySQL Exporter
  - job_name: 'mysql'
    static_configs:
      - targets: ['192.168.1.21:9104']
        labels:
          instance: 'mysql-master'

  # Redis Exporter
  - job_name: 'redis'
    static_configs:
      - targets: ['192.168.1.31:9121']

  # Nginx Exporter (需 nginx-module-vts)
  - job_name: 'nginx'
    static_configs:
      - targets: ['192.168.1.41:9113']

# 安装 node_exporter
wget https://github.com/prometheus/node_exporter/releases/download/v1.8.0/node_exporter-1.8.0.linux-amd64.tar.gz
tar xzf node_exporter-1.8.0.linux-amd64.tar.gz
mv node_exporter-1.8.0.linux-amd64/node_exporter /usr/local/bin/

cat > /etc/systemd/system/node_exporter.service << 'EOF'
[Unit]
Description=Node Exporter
After=network.target

[Service]
User=prometheus
ExecStart=/usr/local/bin/node_exporter \
  --collector.systemd \
  --collector.processes \
  --collector.tcpstat \
  --collector.filesystem.mount-points-exclude='^/(dev|proc|sys|run|var/lib/docker/.+|var/lib/kubelet/.+)' \
  --web.listen-address=:9100

[Install]
WantedBy=multi-user.target
EOF

systemctl daemon-reload && systemctl enable --now node_exporter

# Ubuntu/Debian
apt-get install -y software-properties-common
add-apt-repository "deb https://packages.grafana.com/oss/deb stable main"
wget -q -O - https://packages.grafana.com/gpg.key | apt-key add -
apt-get update && apt-get install -y grafana

# CentOS/RHEL 7/8
cat > /etc/yum.repos.d/grafana.repo << 'EOF'
[grafana]
name=grafana
baseurl=https://packages.grafana.com/oss/rpm
repo_gpgcheck=1
enabled=1
gpgcheck=1
gpgkey=https://packages.grafana.com/gpg.key
sslverify=1
sslcacert=/etc/pki/tls/certs/ca-bundle.crt
EOF
yum install -y grafana

systemctl enable --now grafana-server

# 重置管理员密码
grafana-cli admin reset-admin-password newpassword

# 安装常用插件
grafana-cli plugins install grafana-piechart-panel
grafana-cli plugins install grafana-clock-panel
grafana-cli plugins install vonage-status-panel
systemctl restart grafana-server

# /opt/prometheus/rules/node_alerts.yml
groups:
  - name: node_alerts
    interval: 30s
    rules:
      # 主机宕机
      - alert: NodeDown
        expr: up == 0
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: "主机 {{ $labels.instance }} 宕机"
          description: "主机 {{ $labels.instance }} 已超过 2 分钟不可达"

      # CPU 使用率过高
      - alert: HighCPUUsage
        expr: 100 - (avg by(instance)(rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 90
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "CPU 使用率 > 90%: {{ $labels.instance }}"
          description: "当前值: {{ $value | humanize }}%"

      # 内存使用率
      - alert: HighMemoryUsage
        expr: (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 > 90
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "内存使用率 > 90%: {{ $labels.instance }}"

      # 磁盘使用率
      - alert: HighDiskUsage
        expr: (1 - node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) * 100 > 85
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "磁盘使用率 > 85%: {{ $labels.instance }} /"

      # 磁盘预计填满
      - alert: DiskWillFillIn4Hours
        expr: predict_linear(node_filesystem_avail_bytes{mountpoint="/"}[1h], 4*3600) < 0
        for: 10m
        labels:
          severity: critical
        annotations:
          summary: "磁盘预计 4 小时内填满: {{ $labels.instance }}"

      # 系统负载过高
      - alert: HighSystemLoad
        expr: node_load15 / count without(cpu, mode)(node_cpu_seconds_total{mode="idle"}) > 1.5
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "负载过高 load15/cores > 1.5: {{ $labels.instance }}"

      # 磁盘 IO 饱和
      - alert: DiskIOSaturation
        expr: rate(node_disk_io_time_seconds_total{device=~"sd[a-z]+"}[5m]) > 0.9
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "磁盘 IO 饱和: {{ $labels.instance }} {{ $labels.device }}"

      # 网络错误
      - alert: NetworkErrors
        expr: rate(node_network_receive_errors_total[5m]) > 1
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "网络接口错误: {{ $labels.instance }} {{ $labels.device }}"

      # 内存即将耗尽
      - alert: OutOfMemorySoon
        expr: node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes < 0.05
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: "内存即将耗尽 (< 5%): {{ $labels.instance }}"

      # inode 使用率
      - alert: HighInodeUsage
        expr: (1 - node_filesystem_files_free{mountpoint="/"} / node_filesystem_files{mountpoint="/"}) * 100 > 85
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "inode 使用率 > 85%: {{ $labels.instance }}"

# /opt/alertmanager/alertmanager.yml
global:
  resolve_timeout: 5m
  # 邮件配置
  smtp_smarthost: 'smtp.example.com:587'
  smtp_from: '[email protected]'
  smtp_auth_username: '[email protected]'
  smtp_auth_password: 'password'
  smtp_require_tls: true

# 告警路由
route:
  group_by: ['alertname', 'severity']
  group_wait: 30s
  group_interval: 5m
  repeat_interval: 4h
  receiver: 'default'
  routes:
    - match:
        severity: critical
      receiver: 'critical'
      continue: true
    - match:
        severity: warning
      receiver: 'warning'

# 接收器
receivers:
  - name: 'default'
    email_configs:
      - to: '[email protected]'

  - name: 'critical'
    email_configs:
      - to: '[email protected]'
    webhook_configs:
      # 钉钉
      - url: 'https://oapi.dingtalk.com/robot/send?access_token=xxx'
        send_resolved: true
    # 企业微信
    wechat_configs:
      - corp_id: 'wwxxx'
        to_party: '1'
        agent_id: '1000001'
        api_secret: 'xxx'
        send_resolved: true

  - name: 'warning'
    email_configs:
      - to: '[email protected]'

# 抑制规则 (避免告警风暴)
inhibit_rules:
  - source_match:
      severity: 'critical'
    target_match:
      severity: 'warning'
    equal: ['instance']

# === CPU ===
# CPU 使用率 (%)
100 - (avg by(instance)(rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100)

# 各 CPU 模式占比
avg by(cpu,mode)(rate(node_cpu_seconds_total[5m])) * 100

# === 内存 ===
# 可用内存百分比
(node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100

# 内存使用率
(1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100

# === 磁盘 ===
# 磁盘使用率
(1 - node_filesystem_avail_bytes / node_filesystem_size_bytes) * 100

# 磁盘读写速率 MB/s
rate(node_disk_read_bytes_total[5m]) / 1024 / 1024
rate(node_disk_written_bytes_total[5m]) / 1024 / 1024

# 磁盘 IOPS
rate(node_disk_reads_completed_total[5m])
rate(node_disk_writes_completed_total[5m])

# disk_io_time (IO 繁忙度)
rate(node_disk_io_time_seconds_total[5m]) * 100

# === 网络 ===
# 网络流量 bytes/sec
rate(node_network_receive_bytes_total[5m])
rate(node_network_transmit_bytes_total[5m])

# 网络连接状态
node_netstat_Tcp_CurrEstab

# TCP 重传率
rate(node_netstat_Tcp_RetransSegs[5m]) / rate(node_netstat_Tcp_OutSegs[5m]) * 100

# === 进程 ===
# 打开文件描述符数量
process_open_fds

# === 系统 ===
# 运行时间 (秒)
node_boot_time_seconds

# 预测磁盘空间
predict_linear(node_filesystem_avail_bytes[1h], 24*3600) < 0

# grafana.ini 配置
[smtp]
enabled = true
host = smtp.example.com:587
user = [email protected]
password = password
from_address = [email protected]

[alerting]
enabled = true
execute_alerts = true

# 一键安装
bash <(curl -Ss https://my-netdata.io/kickstart.sh)

# 仅本机访问
sed -i 's/bind to = \*/bind to = 127.0.0.1/g' /etc/netdata/netdata.conf
systemctl restart netdata

# 访问 http://localhost:19999
# 特点: 零配置、极低资源占用、1秒粒度、数千指标自动采集

#!/bin/bash
# /opt/scripts/monitor.sh
# crontab: */5 * * * * /opt/scripts/monitor.sh

HOSTNAME=$(hostname)
ALERT_WEBHOOK="https://hooks.slack.com/services/xxx"

# CPU
CPU_USAGE=$(top -bn1 | grep "Cpu(s)" | awk '{print $2}' | cut -d'%' -f1)
if (( $(echo "$CPU_USAGE > 90" | bc -l) )); then
    curl -X POST -H 'Content-type: application/json' \
        --data "{\"text\":\"[${HOSTNAME}] CPU 使用率: ${CPU_USAGE}%\"}" \
        $ALERT_WEBHOOK
fi

# 内存
MEM_AVAIL=$(free -m | awk 'NR==2{print $7}')
if [ "$MEM_AVAIL" -lt 512 ]; then
    curl -X POST -H 'Content-type: application/json' \
        --data "{\"text\":\"[${HOSTNAME}] 可用内存不足: ${MEM_AVAIL}MB\"}" \
        $ALERT_WEBHOOK
fi

# 磁盘
DISK_USAGE=$(df / | awk 'NR==2{print $5}' | sed 's/%//')
if [ "$DISK_USAGE" -gt 85 ]; then
    curl -X POST -H 'Content-type: application/json' \
        --data "{\"text\":\"[${HOSTNAME}] 磁盘使用率: ${DISK_USAGE}%\"}" \
        $ALERT_WEBHOOK
fi

# 关键进程检查
for proc in nginx mysqld sshd; do
    if ! pgrep -x "$proc" > /dev/null; then
        curl -X POST -H 'Content-type: application/json' \
            --data "{\"text\":\"[${HOSTNAME}] 进程 $proc 未运行!\"}" \
            $ALERT_WEBHOOK
    fi
done

# /etc/rsyslog.conf

# === 模块加载 ===
module(load="imuxsock")     # 本地 socket
module(load="imklog")       # 内核日志
module(load="imtcp")        # TCP 接收
module(load="imudp")        # UDP 接收
module(load="impstats"
       interval="300"
       severity="7"
       log.file="/var/log/rsyslog-stats.log"
       Ruleset="stats")      # 性能统计

# === 全局配置 ===
global(
    workDirectory="/var/lib/rsyslog"
    maxMessageSize="64k"
    defaultTemplate="RSYSLOG_TraditionalFileFormat"
    privDropToUser="syslog"
    privDropToGroup="syslog"
)

# === 日志格式模板 ===
template(name="RemoteLogs" type="string"
    string="%TIMESTAMP% %HOSTNAME% %syslogtag% %msg%\n")

template(name="JsonFormat" type="list") {
    constant(value="{")
    constant(value="\"timestamp\":\"")      property(name="timereported" dateFormat="rfc3339")
    constant(value="\",\"host\":\"")        property(name="hostname")
    constant(value="\",\"severity\":\"")    property(name="syslogseverity")
    constant(value="\",\"facility\":\"")    property(name="syslogfacility")
    constant(value="\",\"tag\":\"")         property(name="syslogtag" format="json")
    constant(value="\",\"message\":\"")     property(name="msg" format="json")
    constant(value="\"}\n")
}

# === 日志规则 ===
# 认证日志
auth,authpriv.*                     /var/log/secure

# 系统日志
*.info;mail.none;authpriv.none     /var/log/messages

# Cron 日志
cron.*                             /var/log/cron

# 内核日志
kern.*                             /var/log/kern.log

# 邮件日志
mail.*                             /var/log/maillog

# Debug 日志 (丢弃)
*.debug                            stop

# 紧急日志发送给所有登录用户
*.emerg                            :omusrmsg:*

# === 转发到远程 ===
# TCP 转发
*.* @@192.168.1.100:514

# UDP 转发
# *.* @192.168.1.100:514

# 条件转发 (仅错误级别以上)
*.err @@192.168.1.100:514

# === 作为日志服务器接收 ===
input(type="imtcp" port="514" Ruleset="remote")
input(type="imudp" port="514" Ruleset="remote")

ruleset(name="remote") {
    # 按主机名分文件
    $template RemotePath,"/data/logs/%HOSTNAME%/%$YEAR%-%$MONTH%-%$DAY%.log"
    action(type="omfile" dynaFile="RemotePath")
    # 也可输出为 JSON
    # action(type="omfile" dynaFile="RemotePath" template="JsonFormat")
}

# Nginx rsyslog 配置
# /etc/rsyslog.d/nginx.conf
$ModLoad imfile
$InputFileName /var/log/nginx/access.log
$InputFileTag nginx-access:
$InputFileStateFile stat-nginx-access
$InputFileSeverity info
$InputFileFacility local7
$InputRunFileMonitor

$InputFileName /var/log/nginx/error.log
$InputFileTag nginx-error:
$InputFileStateFile stat-nginx-error
$InputFileSeverity error
$InputFileFacility local7
$InputRunFileMonitor

local7.* @@192.168.1.100:514

# === 查看日志 ===
journalctl                           # 所有日志
journalctl -n 100                    # 最近 100 行
journalctl -f                        # tail -f 模式
journalctl -k                        # 内核日志
journalctl -u nginx                  # 指定服务
journalctl -u nginx --since today    # 今天的日志
journalctl -u nginx --since "2024-01-01" --until "2024-01-02"
journalctl -p err                     # 仅错误级别以上
journalctl -p emerg..err              # emerg 到 err
journalctl _PID=1234                  # 按 PID
journalctl _UID=0                     # 按 UID (root)
journalctl -o json-pretty             # JSON 输出
journalctl --disk-usage               # 日志占用空间
journalctl -u sshd | grep "Failed"   # 配合 grep

# === journald 配置 ===
# /etc/systemd/journald.conf
[Journal]
Storage=persistent        # 持久化到磁盘
Compress=yes             # 压缩
Seal=yes                 # 防篡改密封
SystemMaxUse=4G          # 最多使用 4G
SystemMaxFileSize=100M   # 单文件最大
MaxRetentionSec=2week    # 最多保留 2 周
RuntimeMaxUse=1G         # /run 下最大使用
ForwardToSyslog=no       # 是否转发到 syslog
ForwardToConsole=no

systemctl restart systemd-journald

# /etc/logrotate.conf (全局)
weekly
rotate 12
create
dateext
compress
include /etc/logrotate.d

# /etc/logrotate.d/nginx (应用级)
/var/log/nginx/*.log {
    daily                       # 每天轮转
    missingok                   # 日志不存在不报错
    rotate 30                   # 保留 30 天
    compress                    # 压缩旧日志
    delaycompress               # 延迟一个周期压缩
    notifempty                  # 空文件不轮转
    create 640 nginx adm        # 创建新文件权限
    sharedscripts               # 轮转完后执行一次脚本
    postrotate
        [ -f /var/run/nginx.pid ] && kill -USR1 $(cat /var/run/nginx.pid)
    endscript
    dateext                     # 日期后缀
    dateformat -%Y%m%d
    maxsize 500M                # 超过 500M 强制轮转
}

# /etc/logrotate.d/syslog
/var/log/cron
/var/log/maillog
/var/log/messages
/var/log/secure
/var/log/spooler
{
    weekly
    rotate 12
    compress
    dateext
    missingok
    sharedscripts
    postrotate
        /bin/kill -HUP $(cat /var/run/syslogd.pid 2>/dev/null) 2>/dev/null || true
    endscript
}

# 手动执行轮转
logrotate -f /etc/logrotate.conf
logrotate -d /etc/logrotate.d/nginx   # 调试模式 (不实际轮转)

# Crontab 中每日执行
# 0 0 * * * /usr/sbin/logrotate /etc/logrotate.conf

# /etc/filebeat/filebeat.yml
filebeat.inputs:
  - type: log
    enabled: true
    paths:
      - /var/log/nginx/access.log
    fields:
      app: nginx
      type: access
    fields_under_root: true
    json.keys_under_root: true
    json.add_error_key: true

  - type: log
    enabled: true
    paths:
      - /var/log/nginx/error.log
    fields:
      app: nginx
      type: error
    fields_under_root: true
    multiline.pattern: '^\d{4}/\d{2}/\d{2}'
    multiline.negate: true
    multiline.match: after

  - type: log
    enabled: true
    paths:
      - /var/log/messages
    fields:
      app: system
      type: syslog

# === 输出到 Logstash ===
output.logstash:
  hosts: ["192.168.1.101:5044"]
  loadbalance: true
  compression_level: 3

# === 或直接输出到 Elasticsearch ===
# output.elasticsearch:
#   hosts: ["192.168.1.101:9200", "192.168.1.102:9200"]
#   index: "filebeat-%{[agent.version]}-%{+yyyy.MM.dd}"

# === 处理器 ===
processors:
  - add_host_metadata: ~
  - add_cloud_metadata: ~
  - drop_fields:
      fields: ["agent.ephemeral_id", "agent.id"]

# === 日志本身 ===
logging.level: info
logging.to_files: true
logging.files:
  path: /var/log/filebeat
  name: filebeat.log
  keepfiles: 7

# /etc/logstash/conf.d/pipeline.conf
input {
  beats {
    port => 5044
    client_inactivity_timeout => 3600
  }
}

filter {
  if [type] == "access" {
    grok {
      match => {
        "message" => '%{IPORHOST:client_ip} - %{DATA:remote_user} \[%{HTTPDATE:timestamp}\] "%{WORD:method} %{DATA:request} HTTP/%{NUMBER:http_version}" %{NUMBER:status} %{NUMBER:body_bytes_sent} "%{DATA:http_referer}" "%{DATA:http_user_agent}"'
      }
    }
    date {
      match => [ "timestamp", "dd/MMM/yyyy:HH:mm:ss Z" ]
      target => "@timestamp"
    }
    geoip {
      source => "client_ip"
    }
    useragent {
      source => "http_user_agent"
      target => "user_agent"
    }
  }

  if [type] == "syslog" {
    grok {
      match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" }
    }
  }
}

output {
  elasticsearch {
    hosts => ["http://192.168.1.101:9200", "http://192.168.1.102:9200"]
    index => "logstash-%{[app]}-%{+YYYY.MM.dd}"
    manage_template => false
  }
  # 如果 ES 不可用，存磁盘队列
  # dead_letter_queue {
  #   path => "/data/logstash/dead_letter_queue"
  #   max_bytes => "1gb"
  # }
}

# promtail (日志采集器) 配置
# /etc/promtail/config.yml
server:
  http_listen_port: 9080

clients:
  - url: http://loki:3100/loki/api/v1/push

scrape_configs:
  - job_name: system
    static_configs:
      - targets: [localhost]
        labels:
          job: varlogs
          host: ${HOSTNAME}
          __path__: /var/log/*.log

  - job_name: nginx
    static_configs:
      - targets: [localhost]
        labels:
          job: nginx
          host: ${HOSTNAME}
          __path__: /var/log/nginx/*.log
    pipeline_stages:
      - match:
          selector: '{job="nginx"} |= "/var/log/nginx/error.log"'
          stages:
            - regex:
                expression: '^(?P<time>\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2}) \[(?P<level>\w+)\]'
            - labels:
                level:

# loki 配置 (单机)
# /etc/loki/loki-config.yaml
auth_enabled: false

server:
  http_listen_port: 3100

ingester:
  lifecycler:
    ring:
      kvstore:
        store: inmemory
      replication_factor: 1
  chunk_idle_period: 30m
  max_chunk_age: 1h
  chunk_target_size: 1536000
  chunk_retain_period: 30s

schema_config:
  configs:
    - from: 2024-01-01
      store: boltdb-shipper
      object_store: filesystem
      schema: v11
      index:
        prefix: index_
        period: 24h

storage_config:
  boltdb_shipper:
    active_index_directory: /data/loki/index
    cache_location: /data/loki/cache
  filesystem:
    directory: /data/loki/chunks

limits_config:
  enforce_metric_name: false
  max_entries_limit_per_query: 5000
  retention_period: 30d

compactor:
  working_directory: /data/loki/compactor
  shared_store: filesystem

# === 基础用法 ===
# 本地同步
rsync -avz --delete /data/ /backup/

# 远程同步 (SSH)
rsync -avz -e "ssh -p 22" /data/ [email protected]:/backup/

# 远程拉取
rsync -avz [email protected]:/data/ /backup/

# === 生产级备份脚本 ===
#!/bin/bash
# /opt/scripts/backup.sh

BACKUP_SRC="/data/app"
BACKUP_DST="/backup"
REMOTE_HOST="[email protected]"
REMOTE_PATH="/backup/$(hostname)"
LOG_FILE="/var/log/backup.log"
EXCLUDE_FILE="/opt/scripts/backup_exclude.txt"
LOCK_FILE="/var/run/backup.lock"

log() {
    echo "[$(date '+%F %T')] $*" | tee -a "$LOG_FILE"
}

# 防止并发执行
exec 200>"$LOCK_FILE"
flock -n 200 || { log "备份已在运行，退出"; exit 1; }

log "=== 开始备份 ==="

# 1. 本地每日快照 (保留 7 天)
rsync -avz \
    --delete \
    --exclude-from="$EXCLUDE_FILE" \
    --link-dest="$BACKUP_DST/latest" \
    "$BACKUP_SRC/" \
    "$BACKUP_DST/$(date +%Y%m%d)/"

# 2. 更新 latest 符号链接
rm -f "$BACKUP_DST/latest"
ln -s "$BACKUP_DST/$(date +%Y%m%d)" "$BACKUP_DST/latest"

# 3. 远程同步
rsync -avz --delete \
    -e "ssh -p 22 -i /root/.ssh/backup_key" \
    "$BACKUP_DST/" "$REMOTE_HOST:$REMOTE_PATH/"

# 4. 清理旧备份 (超过 30 天的远程备份)
ssh -i /root/.ssh/backup_key "$REMOTE_HOST" \
    "find $REMOTE_PATH -maxdepth 1 -type d -mtime +30 -exec rm -rf {} +"

log "=== 备份完成 ==="

# === 安装 ===
wget https://github.com/restic/restic/releases/download/v0.16.4/restic_0.16.4_linux_amd64.bz2
bunzip2 restic_0.16.4_linux_amd64.bz2
mv restic_0.16.4_linux_amd64 /usr/local/bin/restic
chmod +x /usr/local/bin/restic

# === 初始化仓库 ===
export RESTIC_REPOSITORY=/backup/restic
export RESTIC_PASSWORD=your_strong_password
restic init

# === 远程仓库 ===
export RESTIC_REPOSITORY=sftp:backup@storage:/backup/restic
export RESTIC_PASSWORD=your_strong_password
restic init

# S3 兼容 (MinIO / AWS S3)
export AWS_ACCESS_KEY_ID=xxx
export AWS_SECRET_ACCESS_KEY=yyy
export RESTIC_REPOSITORY=s3:s3.amazonaws.com/bucket-name/restic
restic init

# === 备份 ===
restic backup /data /etc /var/log --exclude "*.tmp" --exclude "*.log"

# === 快照管理 ===
restic snapshots                          # 列出快照
restic diff <snapshot1> <snapshot2>       # 比较快照差异
restic stats                              # 仓库统计

# === 恢复 ===
restic restore latest --target /restore/path/    # 恢复最新
restic restore <snapshot_id> --target /restore/  # 恢复指定快照
restic restore latest --target /restore/ --include "/data/app/*"

# === 清理 ===
restic forget --keep-daily 7 --keep-weekly 4 --keep-monthly 12 --keep-yearly 2
restic prune     # 删除未引用的数据块
restic check     # 验证仓库完整性

# === 自动备份脚本 ===
#!/bin/bash
export RESTIC_REPOSITORY=/backup/restic
export RESTIC_PASSWORD_FILE=/root/.restic-password
BACKUP_SRC="/data /etc /var/log"

restic backup $BACKUP_SRC \
    --exclude "*.tmp" \
    --exclude "*.bak" \
    --tag "$(date +%Y%m%d)" \
    --host "$(hostname)"

restic forget \
    --keep-daily 7 \
    --keep-weekly 4 \
    --keep-monthly 12 \
    --keep-yearly 2 \
    --prune

restic check --read-data-subset=2%

#!/bin/bash
# MySQL 全量 + binlog 备份

DB_USER="backup"
DB_PASS="password"
BACKUP_DIR="/backup/mysql"
RETENTION_DAYS=7

# 全量备份 (mysqldump)
mysqldump -u$DB_USER -p$DB_PASS --all-databases \
    --single-transaction \
    --routines --triggers --events \
    --master-data=2 \
    --set-gtid-purged=OFF \
    | gzip > "$BACKUP_DIR/full_$(date +%Y%m%d_%H%M).sql.gz"

# 或使用 xtrabackup (物理备份，大库推荐)
xtrabackup --backup \
    --user=$DB_USER --password=$DB_PASS \
    --target-dir="$BACKUP_DIR/xtra_$(date +%Y%m%d_%H%M)" \
    --compress --compress-threads=4

# 清理旧备份
find "$BACKUP_DIR" -mtime +$RETENTION_DAYS -delete

#!/bin/bash
# PostgreSQL 备份

BACKUP_DIR="/backup/postgres"
export PGPASSWORD="password"

# 逻辑备份
pg_dumpall -U postgres -h localhost | gzip > "$BACKUP_DIR/pg_all_$(date +%Y%m%d).sql.gz"

# 单库自定义格式 (支持并行恢复)
pg_dump -U postgres -h localhost -Fc -j 4 mydb > "$BACKUP_DIR/mydb_$(date +%Y%m%d).dump"

# WAL 归档配置 (postgresql.conf)
# wal_level = replica
# archive_mode = on
# archive_command = 'test ! -f /backup/pg_wal/%f && cp %p /backup/pg_wal/%f'

# === 分区镜像备份 (dd) ===
dd if=/dev/sda1 of=/backup/sda1_$(date +%Y%m%d).img bs=4M status=progress

# 压缩
dd if=/dev/sda1 bs=4M | gzip > /backup/sda1.img.gz

# 恢复
gunzip -c /backup/sda1.img.gz | dd of=/dev/sda1 bs=4M status=progress

# === tar 系统备份 ===
tar -cvpzf /backup/system_$(date +%Y%m%d).tar.gz \
    --exclude=/proc \
    --exclude=/tmp \
    --exclude=/sys \
    --exclude=/dev \
    --exclude=/run \
    --exclude=/mnt \
    --exclude=/media \
    --exclude=/backup \
    --exclude=/lost+found \
    /

# 恢复
tar -xvpzf /backup/system_20240101.tar.gz -C /

#!/bin/bash
# DR 演练检查清单脚本

echo "=== 灾难恢复检查清单 ==="
echo "日期: $(date)"

# 1. 验证备份完整性
echo -e "\n[1/6] 验证备份完整性..."
BACKUP_DIR="/backup"
LATEST=$(ls -t "$BACKUP_DIR" | head -1)
if [ -n "$LATEST" ]; then
    echo "  ✓ 最新备份: $LATEST"
else
    echo "  ✗ 未找到备份!"
fi

# 2. 验证备份可恢复性 (抽检)
echo -e "\n[2/6] 验证备份可恢复性..."
if restic check --read-data-subset=1%; then
    echo "  ✓ restic 仓库完整性验证通过"
fi

# 3. 验证数据库备份
echo -e "\n[3/6] 验证数据库备份..."
LATEST_SQL=$(ls -t /backup/mysql/*.sql.gz 2>/dev/null | head -1)
if gzip -t "$LATEST_SQL" 2>/dev/null; then
    echo "  ✓ MySQL 备份文件完整性验证通过"
fi

# 4. 验证远程同步
echo -e "\n[4/6] 验证远程备份..."
if rsync -azn --delete /backup/ [email protected]:/backup/; then
    echo "  ✓ 远程连接正常"
fi

# 5. 验证恢复文档
echo -e "\n[5/6] 验证恢复文档..."
if [ -f /opt/docs/recovery_procedure.md ]; then
    echo "  ✓ 恢复文档存在"
else
    echo "  ✗ 缺少恢复文档!"
fi

# 6. 验证恢复时间
echo -e "\n[6/6] 恢复时间估算..."
echo "  上次全量恢复耗时: ~30分钟 (记录于 2024-01-01)"
echo "  RTO 目标: 2小时"
echo "  RPO 目标: < 1小时 (binlog 实时同步)"

echo -e "\n=== 检查完成 ==="

# 安装
yum install -y ansible        # CentOS/RHEL
apt-get install -y ansible    # Ubuntu/Debian
pip install ansible            # pip (最新版)

# 验证
ansible --version

# === ansible.cfg 配置 ===
# /etc/ansible/ansible.cfg 或 ./ansible.cfg
[defaults]
inventory      = ./hosts
host_key_checking = False
remote_user    = root
private_key_file = /root/.ssh/id_rsa
forks          = 20
timeout        = 30
log_path       = /var/log/ansible.log
gathering      = smart
fact_caching   = jsonfile
fact_caching_connection = /tmp/ansible_cache
fact_caching_timeout = 3600
retry_files_enabled = False
callback_whitelist = timer, profile_tasks
stdout_callback = yaml

[privilege_escalation]
become         = True
become_method  = sudo
become_user    = root

[ssh_connection]
pipelining     = True
control_path   = /tmp/ansible-%%h-%%p-%%r

# hosts (静态清单)
[webservers]
web01 ansible_host=192.168.1.11
web02 ansible_host=192.168.1.12
web03 ansible_host=192.168.1.13

[dbservers]
db01 ansible_host=192.168.1.21 ansible_user=dbadmin
db02 ansible_host=192.168.1.22

[appservers]
app[01:05].example.com             # 范围: app01 ~ app05

[production:children]              # 分组嵌套
webservers
dbservers

[production:vars]                  # 组变量
ansible_user=root
ntp_server=ntp.prod.example.com

[all:vars]                         # 全局变量
ansible_port=22

# hosts.yml (YAML 清单)
all:
  hosts:
    bastion:
      ansible_host: 1.2.3.4
  children:
    production:
      hosts:
        web[01:03].example.com:
      vars:
        env: production
    staging:
      hosts:
        web-stg.example.com:
      vars:
        env: staging

# 基本语法: ansible <pattern> -m <module> -a "<arguments>"

# === 信息收集 ===
ansible all -m ping                                       # 存活检测
ansible all -m setup                                      # 收集 facts
ansible all -m setup -a "filter=ansible_memory_mb"        # 过滤 facts
ansible all -m shell -a "hostname; uptime"                # 执行 shell

# === 文件操作 ===
ansible all -m copy -a "src=/tmp/file dest=/tmp/file"     # 拷贝文件
ansible all -m fetch -a "src=/etc/hosts dest=/tmp/"       # 拉取文件
ansible all -m file -a "path=/data state=directory mode=0755"  # 创建目录
ansible all -m replace -a "path=/etc/nginx/nginx.conf regexp='worker_processes.*' replace='worker_processes 8;'"  # 文件内容替换

# === 包管理 ===
ansible all -m yum -a "name=nginx state=latest"           # 安装 (RHEL)
ansible all -m apt -a "name=nginx state=latest update_cache=yes"  # 安装 (Debian)

# === 服务管理 ===
ansible all -m systemd -a "name=nginx state=restarted"    # 重启服务
ansible all -m systemd -a "name=nginx enabled=yes"        # 开机启动

# === 用户管理 ===
ansible all -m user -a "name=app password={{ 'mypass' | password_hash('sha512') }} groups=wheel state=present"

# === 内核参数 ===
ansible all -m sysctl -a "name=net.ipv4.tcp_tw_reuse value=1 sysctl_set=yes reload=yes"

# === 计划任务 ===
ansible all -m cron -a "name='log cleanup' hour=2 job='/opt/scripts/cleanup.sh'"

# === 防火墙 ===
ansible all -m firewalld -a "port=80/tcp permanent=yes state=enabled immediate=yes"

# deploy_webapp.yml
---
- name: 部署 Web 应用
  hosts: webservers
  become: yes
  vars:
    app_name: myapp
    app_port: 8080
    app_version: "1.2.3"
    nginx_worker_processes: "{{ ansible_processor_vcpus }}"

  vars_files:
    - vars/secrets.yml   # 加密的敏感变量

  pre_tasks:
    - name: 更新 yum 缓存
      yum:
        update_cache: yes
        name: '*'
        state: latest
      when: ansible_os_family == "RedHat"
      tags: [update]

    - name: 检查磁盘空间
      shell: df -h /data | awk 'NR==2{print $5}' | sed 's/%//'
      register: disk_usage
      failed_when: disk_usage.stdout|int > 90

  tasks:
    - name: 安装基础包
      package:
        name: "{{ item }}"
        state: present
      loop:
        - nginx
        - supervisor
        - python3
      tags: [packages]

    - name: 创建应用用户
      user:
        name: "{{ app_name }}"
        system: yes
        shell: /sbin/nologin
        create_home: no
      tags: [user]

    - name: 创建目录结构
      file:
        path: "{{ item }}"
        state: directory
        owner: "{{ app_name }}"
        mode: '0755'
      loop:
        - /opt/{{ app_name }}
        - /opt/{{ app_name }}/config
        - /data/{{ app_name }}
        - /var/log/{{ app_name }}
      tags: [dirs]

    - name: 部署应用文件
      copy:
        src: files/{{ app_name }}-{{ app_version }}.jar
        dest: /opt/{{ app_name }}/{{ app_name }}.jar
        owner: "{{ app_name }}"
        mode: '0644'
      notify: restart app          # 触发 handler
      tags: [deploy]

    - name: 配置 Nginx 反向代理
      template:
        src: templates/nginx.conf.j2
        dest: /etc/nginx/conf.d/{{ app_name }}.conf
        validate: nginx -t -c %s
      notify: reload nginx
      tags: [nginx]

    - name: 配置 systemd 服务
      template:
        src: templates/app.service.j2
        dest: /etc/systemd/system/{{ app_name }}.service
      notify: restart app
      tags: [service]

    - name: 启动服务
      systemd:
        name: "{{ app_name }}"
        state: started
        enabled: yes
      tags: [service]

  handlers:
    - name: reload nginx
      systemd:
        name: nginx
        state: reloaded

    - name: restart app
      systemd:
        name: "{{ app_name }}"
        state: restarted

  post_tasks:
    - name: 健康检查
      uri:
        url: "http://localhost:{{ app_port }}/health"
        status_code: 200
      retries: 10
      delay: 3
      until: result.status == 200
      register: result
      tags: [verify]

    - name: 发送部署通知
      slack:
        token: "{{ slack_token }}"
        msg: "{{ app_name }} v{{ app_version }} 部署完成 [{{ ansible_hostname }}]"
      tags: [notify]

{# templates/nginx.conf.j2 #}
upstream {{ app_name }}_backend {
{% for host in groups['appservers'] %}
    server {{ hostvars[host]['ansible_host'] }}:{{ app_port }} weight=1 max_fails=3 fail_timeout=30s;
{% endfor %}
    keepalive 32;
}

server {
    listen 80;
    server_name {{ app_name }}.example.com;

    access_log /var/log/nginx/{{ app_name }}_access.log;
    error_log  /var/log/nginx/{{ app_name }}_error.log;

    location / {
        proxy_pass http://{{ app_name }}_backend;
        proxy_http_version 1.1;
        proxy_set_header Connection "";
        proxy_set_header Host $host;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;

        # 基于环境控制超时
        {% if env == 'production' %}
        proxy_read_timeout 60s;
        {% else %}
        proxy_read_timeout 300s;
        {% endif %}
    }

    # 仅生产环境启用 SSL
    {% if env == 'production' %}
    listen 443 ssl;
    ssl_certificate     /etc/ssl/certs/{{ app_name }}.crt;
    ssl_certificate_key /etc/ssl/private/{{ app_name }}.key;
    if ($scheme != "https") {
        return 301 https://$host$request_uri;
    }
    {% endif %}
}

# 创建加密文件
ansible-vault create vars/secrets.yml

# 编辑加密文件
ansible-vault edit vars/secrets.yml

# 加密已有文件
ansible-vault encrypt vars/secrets.yml

# 使用密码文件 (生产环境)
echo "my_vault_password" > ~/.vault_pass
ansible-vault encrypt vars/secrets.yml --vault-password-file ~/.vault_pass

# 运行 playbook 时解密
ansible-playbook deploy.yml --vault-password-file ~/.vault_pass
ansible-playbook deploy.yml --ask-vault-pass   # 交互输入

# 多环境密码 (vault-id)
ansible-vault encrypt --vault-id prod@prompt vars/prod/secrets.yml
ansible-playbook deploy.yml --vault-id prod@prompt

# playbook 中使用 roles
---
- hosts: webservers
  roles:
    - role: nginx
      nginx_port: 8080          # 覆盖默认变量
      tags: [nginx]
    - role: app
      tags: [app]

# === 1. SSH 加固 ===
# /etc/ssh/sshd_config
Port 2222                              # 修改默认端口
Protocol 2                             # 仅 SSHv2
PermitRootLogin no                     # 禁止 root 登录
PasswordAuthentication no              # 禁用密码认证
PubkeyAuthentication yes               # 仅密钥认证
MaxAuthTries 3                         # 最大尝试次数
ClientAliveInterval 300                # 客户端心跳
ClientAliveCountMax 2                  # 推送周期
AllowUsers [email protected].*            # 限制用户和来源 IP
X11Forwarding no                       # 禁止 X11 转发
MaxSessions 5                          # 单连接最大会话数
LoginGraceTime 30                      # 认证超时
MaxStartups 10:30:60                   # 未认证连接限制

systemctl restart sshd

# === 2. 密码策略 ===
# /etc/login.defs
PASS_MAX_DAYS   90     # 密码 90 天过期
PASS_MIN_DAYS   7      # 修改后 7 天内不可再改
PASS_MIN_LEN    12     # 最小 12 位
PASS_WARN_AGE   14     # 过期前 14 天警告

# /etc/security/pwquality.conf
minlen = 12
dcredit = -1            # 至少 1 个数字
ucredit = -1            # 至少 1 个大写
lcredit = -1            # 至少 1 个小写
ocredit = -1            # 至少 1 个特殊字符
minclass = 4            # 至少 4 种字符类别
maxrepeat = 3           # 最多连续重复 3 次
maxclassrepeat = 3      # 同类字符最多连续 3 个
difok = 5               # 新密码与旧密码至少不同 5 个字符
enforce_for_root        # root 也适用

# === 3. 账户锁定 ===
# /etc/pam.d/sshd
# 连续 5 次失败后锁定 600 秒
auth required pam_tally2.so deny=5 unlock_time=600 onerr=fail audit

# === 4. 会话超时 ===
echo "TMOUT=600" >> /etc/profile
echo "readonly TMOUT" >> /etc/profile
echo "export TMOUT" >> /etc/profile

# === 5. 历史命令限制 ===
echo 'HISTSIZE=500' >> /etc/profile
echo 'HISTFILESIZE=500' >> /etc/profile
echo "readonly HISTSIZE HISTFILESIZE" >> /etc/profile
echo 'export HISTTIMEFORMAT="%F %T "' >> /etc/profile

# === 6. 限制 su/sudo ===
# 仅 wheel 组可 su
# /etc/pam.d/su
auth required pam_wheel.so use_uid

# sudo 日志审计
# /etc/sudoers
Defaults logfile=/var/log/sudo.log
Defaults log_input,log_output     # 记录输入输出 (需 sudo 1.9+)
Defaults requiretty               # 必须有 tty

# === 基础操作 ===
systemctl start firewalld
systemctl enable firewalld
firewall-cmd --state

# 查看
firewall-cmd --list-all                    # 当前区域详情
firewall-cmd --get-default-zone            # 默认区域
firewall-cmd --get-active-zones            # 活动区域
firewall-cmd --list-services              # 已允许服务
firewall-cmd --list-ports                 # 已允许端口

# 规则管理
firewall-cmd --add-port=8080/tcp --permanent    # 永久开放端口
firewall-cmd --add-service=http --permanent      # 开放服务
firewall-cmd --add-rich-rule='rule family="ipv4" source address="192.168.1.0/24" port port="22" protocol="tcp" accept' --permanent  # 仅允许特定 IP 段访问 SSH
firewall-cmd --remove-port=8080/tcp --permanent  # 删除

# 重载
firewall-cmd --reload

# 区域切换
firewall-cmd --set-default-zone=dmz
firewall-cmd --change-interface=ens33 --zone=trusted --permanent

# === 生产级规则示例 ===
# 默认拒绝并只开放必要端口
firewall-cmd --set-default-zone=drop

firewall-cmd --permanent --add-rich-rule='rule family="ipv4" source address="10.0.0.0/8" port port="22" protocol="tcp" accept'
firewall-cmd --permanent --add-service=http
firewall-cmd --permanent --add-service=https
firewall-cmd --permanent --add-port=9100/tcp    # node_exporter
firewall-cmd --reload

# === 端口转发 (NAT) ===
firewall-cmd --permanent --add-masquerade
firewall-cmd --permanent --add-forward-port=port=80:proto=tcp:toport=8080:toaddr=192.168.1.100
firewall-cmd --reload

# === 默认策略 ===
iptables -P INPUT DROP
iptables -P FORWARD DROP
iptables -P OUTPUT ACCEPT

# === 允许回环 ===
iptables -A INPUT -i lo -j ACCEPT

# === 允许已建立连接 ===
iptables -A INPUT -m state --state ESTABLISHED,RELATED -j ACCEPT

# === 允许 SSH ===
iptables -A INPUT -p tcp --dport 22 -s 10.0.0.0/8 -j ACCEPT

# === 允许 Web ===
iptables -A INPUT -p tcp --dport 80 -j ACCEPT
iptables -A INPUT -p tcp --dport 443 -j ACCEPT

# === 允许 ICMP (ping) ===
iptables -A INPUT -p icmp --icmp-type echo-request -j ACCEPT

# === 防 DDoS ===
# 限制 SYN 包速率
iptables -A INPUT -p tcp --syn -m limit --limit 10/s --limit-burst 20 -j ACCEPT
iptables -A INPUT -p tcp --syn -j DROP

# 限制单个 IP 并发连接
iptables -A INPUT -p tcp --dport 80 -m connlimit --connlimit-above 50 -j DROP

# === 端口转发 ===
iptables -t nat -A PREROUTING -p tcp --dport 80 -j DNAT --to-destination 192.168.1.100:8080
iptables -t nat -A POSTROUTING -j MASQUERADE

# === 保存规则 ===
iptables-save > /etc/sysconfig/iptables       # CentOS 6
iptables-save > /etc/iptables/rules.v4        # Debian/Ubuntu
netfilter-persistent save                      # iptables-persistent

# === 状态查看 ===
getenforce                          # 查看模式
sestatus -v                         # 详细状态

# === 模式切换 ===
setenforce 0                        # 临时切换为 Permissive
setenforce 1                        # 启用 Enforcing

# 永久配置
# /etc/selinux/config
# SELINUX=enforcing|permissive|disabled

# === 上下文管理 ===
ls -Z /var/www/html/                # 查看文件上下文
ps -Z                               # 查看进程上下文

chcon -R -t httpd_sys_content_t /var/www/html/    # 修改上下文
restorecon -Rv /var/www/html/                      # 恢复默认上下文

semanage fcontext -a -t httpd_sys_content_t "/web(/.*)?"
restorecon -Rv /web

# === 布尔值管理 ===
getsebool -a                        # 列出所有布尔值
setsebool -P httpd_can_network_connect on   # 允许 Apache 网络连接
setsebool -P httpd_enable_homedirs on       # 允许用户目录

# === 端口管理 ===
semanage port -l                          # 列出所有端口
semanage port -a -t http_port_t -p tcp 8080  # 添加端口到类型

# === 审计日志排错 ===
ausearch -m avc -ts recent           # 查看最近的 AVC 拒绝
sealert -a /var/log/audit/audit.log  # 分析审计日志
audit2allow -a -M mypol              # 从审计日志生成策略模块
semodule -i mypol.pp                 # 安装自定义模块

# === SELinux 排错流程 ===
# 1. 查看审计日志
ausearch -m avc -ts today | grep denied
# 2. 分析并提供建议
audit2why < /var/log/audit/audit.log
# 3. 临时切换 permissive 排查
setenforce 0
# 4. 测试应用
# 5. 查看生成的 AVC
ausearch -m avc -ts recent
# 6. 创建自定义策略
grep denied /var/log/audit/audit.log | audit2allow -M custom_policy
semodule -i custom_policy.pp
# 7. 恢复 enforcing
setenforce 1

# 安装
yum install -y fail2ban      # CentOS
apt-get install -y fail2ban  # Ubuntu

# /etc/fail2ban/jail.local
[DEFAULT]
ignoreip = 127.0.0.1/8 10.0.0.0/8 192.168.0.0/16
bantime  = 3600                    # 封禁时间 (秒)
findtime = 600                     # 统计窗口 (秒)
maxretry = 5                       # 最大失败次数
destemail = [email protected]
action = %(action_mw)s             # 封禁 + whois + 邮件
banaction = iptables-multiport

[sshd]
enabled  = true
port     = ssh,2222
logpath  = %(sshd_log)s
maxretry = 3

[nginx-http-auth]
enabled  = true
port     = http,https
logpath  = /var/log/nginx/error.log
maxretry = 5

[nginx-botsearch]
enabled  = true
port     = http,https
logpath  = /var/log/nginx/access.log
maxretry = 3
findtime = 300

[mysqld-auth]
enabled  = true
port     = 3306
logpath  = /var/log/mysql/error.log
maxretry = 5

# 管理命令
fail2ban-client status                       # 查看状态
fail2ban-client status sshd                  # 查看 sshd jail
fail2ban-client set sshd unbanip 1.2.3.4     # 手动解封
fail2ban-client set sshd banip 1.2.3.4       # 手动封禁
fail2ban-client reload                       # 重载配置

# 查看封禁日志
grep "Ban" /var/log/fail2ban.log

# 安装
yum install -y audit             # CentOS
apt-get install -y auditd        # Ubuntu

systemctl enable --now auditd

# === 审计规则 ===
# /etc/audit/rules.d/audit.rules

# 监控关键文件
-w /etc/passwd -p wa -k identity_changes
-w /etc/shadow -p wa -k identity_changes
-w /etc/group -p wa -k identity_changes
-w /etc/sudoers -p wa -k sudo_changes
-w /etc/ssh/sshd_config -p wa -k sshd_config
-w /etc/crontab -p wa -k cron_changes

# 监控关键命令执行
-a always,exit -F path=/usr/bin/su -F perm=x -k su_exec
-a always,exit -F path=/usr/bin/sudo -F perm=x -k sudo_exec

# 监控系统调用
-a always,exit -F arch=b64 -S execve -k command_exec

# 监控网络配置修改
-a always,exit -F path=/sbin/ifconfig -F perm=x -k net_config

# 监控时间修改
-a always,exit -F arch=b64 -S adjtimex -S settimeofday -k time_change
-a always,exit -F arch=b64 -S clock_settime -k time_change

# === 审计查询 ===
ausearch -k identity_changes            # 按 key 查询
ausearch -f /etc/passwd                 # 按文件查询
ausearch -p 1234                        # 按 PID 查询
ausearch -ua root                       # 按用户查询
ausearch -ts today                      # 今天的审计记录
ausearch -m USER_LOGIN                  # 登录事件

# 生成报告
aureport -l                             # 登录报告
aureport -k                             # key 汇总
aureport -f                             # 文件审计报告
aureport --summary                      # 摘要

# 搜索失败事件
ausearch -m USER_LOGIN --success no