Nginx
故障排查手册

# 1. 看 error.log 找具体报错(关键!)
tail -f /var/log/nginx/error.log

# 典型错误一:连接被拒绝(后端进程没启动)
# connect() failed (111: Connection refused) while connecting to upstream

# 典型错误二:无可用上游(所有后端都不健康)
# no live upstreams while connecting to upstream

# 典型错误三:upstream 提前关闭连接
# upstream prematurely closed connection while reading response header

# 2. 直连后端确认
curl -v http://<upstream-ip>:<port>/

# 3. 检查后端进程
ss -lntp | grep <port>
ps -ef | grep <service>

# 4. 检查 Nginx 上下游配置
nginx -T | grep -A 5 upstream

# 错误:后端是 HTTPS 但 Nginx 用 http 转发
location /api/ {
    proxy_pass http://backend;    # 后端实际是 https → 502
}

# 正确:后端 HTTPS 必须用 https,且配置证书验证策略
location /api/ {
    proxy_pass https://backend;
    proxy_ssl_server_name on;
    proxy_ssl_verify off;          # 内网自签证书可关闭验证
}

# error.log 典型报错
# upstream timed out (110: Connection timed out) while reading response header

# 看是哪个阶段超时
grep "timed out" /var/log/nginx/error.log | tail

location /api/ {
    proxy_pass http://backend;

    proxy_connect_timeout 5s;     # 与后端建立连接的超时(默认 60s)
    proxy_send_timeout 60s;       # 向后端发送请求的超时
    proxy_read_timeout 60s;       # 等后端响应的超时(默认 60s)
                                       # 长接口/大文件下载必调!
}

# 1. 后端是否慢?直连后端测试时延
time curl http://backend:8080/slow-api

# 2. 看后端日志找慢查询(MySQL slow log / 应用 APM)

# 3. 看后端是否资源不足(CPU/内存/IO)
top
iostat -x 1

# 看 499 占比
awk '{print $9}' /var/log/nginx/access.log | sort | uniq -c | sort -rn

# 看哪些接口 499 多
awk '$9==499 {print $7}' /var/log/nginx/access.log | sort | uniq -c | sort -rn | head

# 看 499 请求的响应时间分布
awk '$9==499 {print $NF}' /var/log/nginx/access.log | sort -n | tail

# 默认行为:客户端断了,Nginx 立即放弃后端请求
# 高并发场景下会导致后端请求大量取消,反而浪费资源

location /api/ {
    proxy_ignore_client_abort on;    # 客户端断开后,Nginx 继续等后端完成
    proxy_pass http://backend;
}
# 适用场景:幂等的写请求、需要完整执行的任务
# 不适用:GET 查询(浪费资源)

upstream backend {
    # 加权 + 健康检查
    server 10.0.0.1:8080 weight=3 max_fails=3 fail_timeout=30s;
    server 10.0.0.2:8080 weight=1;
    server 10.0.0.3:8080 backup;          # 主全挂了才启用
    server 10.0.0.4:8080 down;            # 暂时下线,不参与

    # 复用与后端的长连接(性能关键!)
    keepalive 32;
    keepalive_timeout 60s;
    keepalive_requests 1000;
}

server {
    location / {
        proxy_pass http://backend;
        proxy_http_version 1.1;             # keepalive 需要 HTTP/1.1
        proxy_set_header Connection "";       # 清除 close 头
    }
}

# max_fails=3 fail_timeout=30s 的含义:
# 在 30 秒内失败 3 次,把该后端标记为 down,持续 30 秒不再分配请求
# 30 秒后,Nginx 会尝试再次发送请求,成功就恢复

# 注意:这是被动健康检查,只在有请求时才检测
# 主动健康检查(定时探测)是商业版功能,开源版需要用 nginx_upstream_check_module

# 主配置 /etc/nginx/nginx.conf

worker_processes auto;            # 自动等于 CPU 核数,推荐!
worker_cpu_affinity auto;         # 进程绑核,减少上下文切换

events {
    worker_connections 65535;     # 每个 worker 最大并发连接数
    use epoll;                      # Linux 必选
    multi_accept on;               # 一次 accept 多个连接
}

# 系统级 fd 限制必须配合调高
worker_rlimit_nofile 65535;

最大并发 = worker_processes × worker_connections
例如 8 核 + 65535 → 理论 52 万并发(实际受系统限制)

# 查看 worker 数量
ps -ef | grep "nginx: worker"

# 看每个 worker 的 CPU 使用
top -p $(pgrep -d, "nginx: worker")

# 看 worker 在做什么(找出问题 worker)
strace -p <worker-pid> -c

ssl_session_cache shared:SSL:50m;     # SSL 会话复用,大幅减少握手
ssl_session_timeout 1d;
ssl_session_tickets on;

# 优先使用快速的椭圆曲线
ssl_protocols TLSv1.2 TLSv1.3;
ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256;
ssl_prefer_server_ciphers off;

# 标准三步:测、重载、验证

# 1. 必须先测试配置语法
nginx -t
# 显示 syntax is ok 和 test is successful 才能继续

# 2. 平滑重载(不中断现有连接)
nginx -s reload
# 或
systemctl reload nginx

# 3. 验证新进程
ps -ef | grep nginx
# 应该看到 master 和新的 worker,旧 worker 在 "shutting down"

# 4. 看 error.log 是否有警告
tail -20 /var/log/nginx/error.log

# 场景一:语法错误
nginx: [emerg] unexpected "}" in /etc/nginx/conf.d/api.conf:23
# → 修配置,nginx -t 通过后再 reload

# 场景二:端口已被占用(只在 start 时出现,reload 不会)
bind() to 0.0.0.0:80 failed (98: Address already in use)

# 场景三:upstream 解析失败
host not found in upstream "api.internal"
# → DNS 不通或 hosts 没配,nginx -t 时就会报

# 场景四:include 的文件不存在
open() "/etc/nginx/conf.d/missing.conf" failed (2: No such file or directory)

# 1. 确认 nginx -t 加载的是哪个配置文件
nginx -t 2>&1 | grep file

# 2. 看完整生效配置(把所有 include 展开)
nginx -T | less

# 3. 确认 reload 真的执行了 — 看 master 进程时间
ps -eo pid,etime,cmd | grep "nginx: master"

# 4. 旧 worker 长期不退出?客户端长连接占着
ps -ef | grep "shutting down"
# 如果有,可用 worker_shutdown_timeout 强制超时

http {
    # 1. 请求体大小限制(默认 1M,常见坑!)
    client_max_body_size 100m;

    # 2. 客户端发送请求体的超时(默认 60s,大文件不够)
    client_body_timeout 600s;

    # 3. 请求体缓冲区(超过会写临时文件)
    client_body_buffer_size 1m;       # 内存缓冲
    client_body_temp_path /var/cache/nginx/client_temp;
}

server {
    location /upload {
        proxy_pass http://backend;

        # 4. 后端代理超时(上传后还要等后端处理)
        proxy_send_timeout 600s;
        proxy_read_timeout 600s;

        # 5. 大文件建议关闭缓冲,直接流式转发
        proxy_request_buffering off;
        proxy_buffering off;
    }
}

# 1. 看证书过期时间
openssl x509 -in /etc/nginx/ssl/cert.pem -noout -dates

# 2. 看证书包含的域名(SAN)
openssl x509 -in cert.pem -noout -text | grep -A 1 "Subject Alternative Name"

# 3. 远程测试证书
openssl s_client -connect example.com:443 -servername example.com < /dev/null

# 4. 一行看到期时间
echo | openssl s_client -connect example.com:443 2>/dev/null | openssl x509 -noout -dates

# 5. 验证证书链完整性
openssl verify -CAfile chain.pem cert.pem

server {
    listen 443 ssl http2;
    server_name example.com;

    # 证书路径(cert.pem 必须包含完整证书链,不只是站点证书)
    ssl_certificate     /etc/nginx/ssl/fullchain.pem;
    ssl_certificate_key /etc/nginx/ssl/private.key;

    # 协议与套件
    ssl_protocols TLSv1.2 TLSv1.3;
    ssl_ciphers ECDHE+AESGCM:ECDHE+CHACHA20:DHE+AESGCM;
    ssl_prefer_server_ciphers off;

    # 会话复用(性能)
    ssl_session_cache shared:SSL:50m;
    ssl_session_timeout 1d;

    # HSTS,强制 HTTPS
    add_header Strict-Transport-Security "max-age=63072000" always;
}

# HTTP 自动跳 HTTPS
server {
    listen 80;
    server_name example.com;
    return 301 https://$host$request_uri;
}

http {
    # 定义限流区域:按客户端 IP,10M 内存,平均 10 req/s
    limit_req_zone $binary_remote_addr zone=api_limit:10m rate=10r/s;

    # 也可按其他变量限,如登录接口按用户名
    limit_req_zone $arg_username zone=login:10m rate=1r/s;
}

server {
    location /api/ {
        # 应用限流,burst 是突发容忍,超过 rate 但在 burst 内会排队
        limit_req zone=api_limit burst=20 nodelay;

        # nodelay 含义:burst 内的请求立即处理,不排队
        # 不加 nodelay:超过 rate 的请求会被强行延迟到匹配 rate

        # 自定义被限流时的响应码(默认 503)
        limit_req_status 429;
    }

    location /login {
        limit_req zone=login burst=5 nodelay;
        limit_req_status 429;
    }
}

http {
    limit_conn_zone $binary_remote_addr zone=conn_per_ip:10m;
    limit_conn_zone $server_name zone=conn_per_server:10m;
}

server {
    location /download/ {
        # 每个 IP 最多 3 个并发下载
        limit_conn conn_per_ip 3;

        # 每个 IP 限速 500K/s
        limit_rate 500k;
    }
}

# =========== 主配置层 ===========
worker_processes     auto;             # 等于 CPU 核数
worker_cpu_affinity  auto;             # 绑核
worker_rlimit_nofile 65535;            # 单 worker 最大 fd

events {
    worker_connections 65535;            # 单 worker 最大连接数
    use epoll;                            # Linux IO 多路复用
    multi_accept on;
}

http {
    # =========== TCP 优化 ===========
    sendfile on;                          # 零拷贝
    tcp_nopush on;                        # 配合 sendfile,合并小包
    tcp_nodelay on;                       # 长连接小数据立即发送

    # =========== 客户端长连接 ===========
    keepalive_timeout 65;
    keepalive_requests 10000;             # 单连接最大请求数

    # =========== Gzip 压缩 ===========
    gzip on;
    gzip_vary on;
    gzip_min_length 1024;                 # 小于 1K 不压缩
    gzip_comp_level 5;                    # 1-9,5 是性价比最高
    gzip_types text/plain application/json application/javascript text/css;

    # =========== 缓冲区 ===========
    client_body_buffer_size    1m;
    client_header_buffer_size  4k;
    large_client_header_buffers 4 16k;    # 大 cookie / URL

    # =========== 隐藏版本 ===========
    server_tokens off;

    # =========== 日志 ===========
    access_log /var/log/nginx/access.log main buffer=32k flush=5s;
}

# /etc/sysctl.conf 必调

# 端口范围
net.ipv4.ip_local_port_range = 1024 65535

# TIME_WAIT 复用
net.ipv4.tcp_tw_reuse = 1

# SYN 队列与 backlog
net.core.somaxconn = 65535
net.ipv4.tcp_max_syn_backlog = 65535

# 全连接队列
net.core.netdev_max_backlog = 65535

# keepalive 探测
net.ipv4.tcp_keepalive_time = 600
net.ipv4.tcp_keepalive_intvl = 30
net.ipv4.tcp_keepalive_probes = 3

# 应用后 sysctl -p

# 1. QPS 实时统计
tail -f access.log | awk '{print $4}' | uniq -c

# 2. Top 10 慢接口
awk '{print $NF, $7}' access.log | sort -rn | head

# 3. 状态码分布
awk '{print $9}' access.log | sort | uniq -c | sort -rn

# 4. 访问最多的 IP(找异常爬虫)
awk '{print $1}' access.log | sort | uniq -c | sort -rn | head

# 5. 5xx 错误聚合
awk '$9 ~ /^5/ {print $7}' access.log | sort | uniq -c | sort -rn