深夜处理 bug：重启 openresty（nginx）后， 403 forbidden，连服务器 ip 都无法访问了

[kmdd33]

手贱重启了 openresty，突然站点就 403 forbidden。搞了一天了，真心累，请教同学们。

配置环境:centos7+openresty(nginx)+php+mysql systemctl status openresty ,systemctl status php-fpm,systemctl status mysql，都是 active status，绿色。

lsof -i :80

COMMAND PID USER FD TYPE DEVICE SIZE/OFF NODE NAME

nginx 32255 root 6u IPv4 170338 0t0 TCP localhost:http (LISTEN)

nginx 32332 root 6u IPv4 170338 0t0 TCP localhost:http (LISTEN)

netstat -an |grep 80

tcp 0 0 127.0.0.1:80 0.0.0.0:* LISTEN

unix 2 [ ACC ] STREAM LISTENING 28071 /var/lib/mysql/mysql.sock

unix 3 [ ] STREAM CONNECTED 11800

unix 3 [ ] STREAM CONNECTED 11801 /run/systemd/journal/stdout

ps aux | grep php

root 19363 0.0 3.1 470292 31572 ? Ss 00:20 0:04 php-fpm: master process (/etc/ph-fpm.conf)

nginx 19364 0.0 0.7 470292 7228 ? S 00:20 0:00 php-fpm: pool www

nginx 19365 0.0 0.7 470292 7228 ? S 00:20 0:00 php-fpm: pool www

nginx 19366 0.0 0.7 470292 7228 ? S 00:20 0:00 php-fpm: pool www

nginx 19367 0.0 0.7 470292 7228 ? S 00:20 0:00 php-fpm: pool www

nginx 19368 0.0 0.7 470292 7232 ? S 00:20 0:00 php-fpm: pool www

nginx 19369 0.0 0.7 470292 7232 ? S 00:20 0:00 php-fpm: pool www

nginx 19370 0.0 0.7 470292 7232 ? S 00:20 0:00 php-fpm: pool www

nginx 19371 0.0 0.7 470292 7232 ? S 00:20 0:00 php-fpm: pool www

root 32204 0.0 0.0 112704 968 pts/0 R+ 17:57 0:00 grep --color=auto php

netstat -plant

Active Internet connections (servers and established)

Proto Recv-Q Send-Q Local Address Foreign Address State PID/Program name

tcp 0 0 127.0.0.1:80 0.0.0.0:* LISTEN 32255/nginx: master

tcp 0 0 0.0.0.0:22 0.0.0.0:* LISTEN 730/sshd

tcp 0 0 45.79.66.57:22 113.118.253.226:4315 ESTABLISHED 32434/sshd: root@pt

tcp 0 36 45.79.66.57:22 113.118.253.226:4860 ESTABLISHED 32584/sshd: root@pt

tcp 0 0 45.79.66.57:22 113.118.253.226:3505 ESTABLISHED 32115/sshd: root@pt

tcp6 0 0 :::9000 :::* LISTEN 19363/php-fpm: mast

tcp6 0 0 :::3306 :::* LISTEN 3299/mysqld

tcp6 0 0 :::22 :::* LISTEN 730/sshd

/usr/local/openresty/nginx/conf/nginx.conf

user root;

worker_processes 1;

#error_log logs/error.log;

#error_log logs/error.log notice;

#error_log logs/error.log info;

#pid logs/nginx.pid;

events { worker_connections 1024; }

http { include mime.types; default_type application/octet-stream;

#log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
#                  '$status $body_bytes_sent "$http_referer" '
#                  '"$http_user_agent" "$http_x_forwarded_for"';

#access_log  logs/access.log  main;

sendfile        on;
#tcp_nopush     on;

#keepalive_timeout  0;
keepalive_timeout  65;

client_max_body_size 8m;    #允许客户端请求的最大单文件字节数
client_body_buffer_size 2m;  #缓冲区代理缓冲用户端请求的最大字节

fastcgi_buffers 8 16k;
fastcgi_buffer_size 32k;
fastcgi_connect_timeout 300;
fastcgi_send_timeout 300;
fastcgi_read_timeout 300;

gzip  on;
#WAF
lua_shared_dict limit 50m;
lua_shared_dict guard_dict 100m;
lua_shared_dict dict_captcha 70m;
lua_package_path "/usr/local/openresty/nginx/conf/waf/?.lua";
init_by_lua_file "/usr/local/openresty/nginx/conf/waf/init.lua";
access_by_lua_file "/usr/local/openresty/nginx/conf/waf/access.lua";

server {
    listen       127.0.0.1;
    server_name  kiwikiwifly.com;

    #charset koi8-r;

    #access_log  logs/host.access.log  main;

    location / {
        root   html;
        index  index.php index.html index.htm;
    }

    #error_page  404              /404.html;

    # redirect server error pages to the static page /50x.html
    #
    error_page   500 502 503 504  /50x.html;
    location = /50x.html {
        root   html;
    }

    # proxy the PHP scripts to Apache listening on 127.0.0.1:80
    #
    #location ~ \.php$ {
    #    proxy_pass   http://127.0.0.1;
    #}

    # pass the PHP scripts to FastCGI server listening on 127.0.0.1:9000
    #
    
    location ~ \.php$ {
        root           html;
        fastcgi_pass   127.0.0.1:9000;
        fastcgi_index  index.php;
        fastcgi_param  SCRIPT_FILENAME  $document_root$fastcgi_script_name;
        include        fastcgi_params;
    }

    # deny access to .htaccess files, if Apache's document root
    # concurs with nginx's one
    #
    #location ~ /\.ht {
    #    deny  all;
    #}
}


# another virtual host using mix of IP-, name-, and port-based configuration
#
#server {
#    listen       8000;
#    listen       somename:8080;
#    server_name  somename  alias  another.alias;

#    location / {
#        root   html;
#        index  index.html index.htm;
#    }
#}


# HTTPS server
#
#server {
#    listen       443 ssl;
#    server_name  localhost;

#    ssl_certificate      cert.pem;
#    ssl_certificate_key  cert.key;

#    ssl_session_cache    shared:SSL:1m;
#    ssl_session_timeout  5m;

#    ssl_ciphers  HIGH:!aNULL:!MD5;
#    ssl_prefer_server_ciphers  on;

#    location / {
#        root   html;
#        index  index.html index.htm;
#    }
#}

}

and /usr/local/openresty/nginx/logs/error.log

2018/06/26 21:27:10 [alert] 740#740: *7927 open socket #10 left in connection 12

2018/06/26 21:27:10 [alert] 740#740: *7926 open socket #3 left in connection 13

2018/06/26 21:27:10 [alert] 740#740: aborting

只有到 26 号 error.log

[Livid]

1. 给 server 那段单独写一个 error_log
2. 试试把 WAF 相关的配置先注释掉

[kmdd33]

@Livid 2.已经注释了，1 给 server 那段单独写一个 error_log ？请问这个如何操作？

[yaxin]

看一下 nginx 有没有 web 根路径的读权限，如果有，在根路径下创建一个 xxxx.php 文件，里面随便输出点字符串，然后访问，看能否访问，能，说明你站点 php 逻辑问题，不能，nginx 问题，通过日志把问题，还是找不到，直接上 strace

[est]

livid 的办法是正解。

开 error_log，debug 级别。就能看到为啥出错了。

[sajesemuy]

你这边怎么又是 403，另一个帖子 502。listen 127.0.0.1，你是直接 在本地 curl 127.0.0.1 出现 403/502 ？
如果是访问你的站点的公网地址，前面那一层怎么访问 127.0.0.1 ？

[wqsfree]

user nobody nobody;
worker_processes 2;
error_log /var/log/nginx/error.log debug ; 这是错误日志的路径，会了么？
pid /var/run/nginx/nginx.pid;

[kmdd33]

@Livid @sajesemuy @est Sorry,Sorry,是 502 bad gateway 的提示。

[kmdd33]

@wqsfree

你好，按照您的建议，把你提供的这个写在了 http 段外面，重新 reload openresty 成功，error_log /var/log/nginx/error.log 内容:

2018/06/29 09:06:08 [notice] 10008#10008: using the "epoll" event method

2018/06/29 09:06:08 [notice] 10008#10008: openresty/1.13.6.2

2018/06/29 09:06:08 [notice] 10008#10008: built by gcc 4.8.5 20150623 (Red Hat 4.8.5-16) (GCC)

2018/06/29 09:06:08 [notice] 10008#10008: OS: Linux 3.10.0-862.3.2.el7.x86_64

2018/06/29 09:06:08 [notice] 10008#10008: getrlimit(RLIMIT_NOFILE): 1024:4096

2018/06/29 09:06:08 [notice] 10009#10009: start worker processes

2018/06/29 09:06:08 [notice] 10009#10009: start worker process 10011

2018/06/29 09:14:52 [notice] 10009#10009: signal 1 (SIGHUP) received from 10206, reconfiguring

2018/06/29 09:14:52 [notice] 10009#10009: reconfiguring

2018/06/29 09:14:52 [notice] 10009#10009: using the "epoll" event method

2018/06/29 09:14:52 [notice] 10009#10009: start worker processes

2018/06/29 09:14:52 [notice] 10009#10009: start worker process 10207

2018/06/29 09:14:52 [notice] 10011#10011: gracefully shutting down

2018/06/29 09:14:52 [notice] 10011#10011: exiting

2018/06/29 09:14:52 [notice] 10011#10011: exit

2018/06/29 09:14:52 [notice] 10009#10009: signal 17 (SIGCHLD) received from 10011

2018/06/29 09:14:52 [notice] 10009#10009: worker process 10011 exited with code 0

2018/06/29 09:14:52 [notice] 10009#10009: signal 29 (SIGIO) received

2018/06/29 09:14:58 [notice] 10009#10009: signal 1 (SIGHUP) received from 10209, reconfiguring

2018/06/29 09:14:58 [notice] 10009#10009: reconfiguring

2018/06/29 09:14:58 [notice] 10009#10009: using the "epoll" event method

2018/06/29 09:14:58 [notice] 10009#10009: start worker processes

2018/06/29 09:14:58 [notice] 10009#10009: start worker process 10210

2018/06/29 09:14:58 [notice] 10207#10207: gracefully shutting down

2018/06/29 09:14:58 [notice] 10207#10207: exiting

2018/06/29 09:14:58 [notice] 10207#10207: exit

2018/06/29 09:14:58 [notice] 10009#10009: signal 17 (SIGCHLD) received from 10207

2018/06/29 09:14:58 [notice] 10009#10009: worker process 10207 exited with code 0

2018/06/29 09:14:58 [notice] 10009#10009: signal 29 (SIGIO) received

[kmdd33]

@wqsfree
/var/run/nginx/nginx.pid 仅仅输出了

10009

这个提示。请问我应该修改什么地方呢？服务器 ip 还是无法访问

[kmdd33]

@Livid @est @sajesemuy @wqsfree @yaxin 最新的 /usr/local/openresty/nginx/log/error.log


8/06/28 18:04:18 [notice] 32331#32331: signal process started
2018/06/28 22:19:32 [emerg] 2711#2711: bind() to 127.0.0.1:80 failed (98: Address already in use)
2018/06/28 22:19:32 [emerg] 2711#2711: bind() to 127.0.0.1:80 failed (98: Address already in use)
2018/06/28 22:19:32 [emerg] 2711#2711: bind() to 127.0.0.1:80 failed (98: Address already in use)
2018/06/28 22:19:32 [emerg] 2711#2711: bind() to 127.0.0.1:80 failed (98: Address already in use)
2018/06/28 22:19:32 [emerg] 2711#2711: bind() to 127.0.0.1:80 failed (98: Address already in use)
2018/06/28 22:19:32 [emerg] 2711#2711: still could not bind()
2018/06/29 08:38:40 [emerg] 9553#9553: open() "/var/run/nginx/nginx.pid" failed (2: No such file or directory)
2018/06/29 08:39:35 [emerg] 9589#9589: open() "/var/run/nginx/nginx.pid" failed (2: No such file or directory)
2018/06/29 08:51:43 [emerg] 9757#9757: open() "/var/log/nginx/error.log" failed (2: No such file or directory)
2018/06/29 08:55:53 [notice] 9801#9801: signal process started
2018/06/29 08:55:53 [error] 9801#9801: invalid PID number "" in "/var/run/nginx/nginx.pid"
2018/06/29 09:09:34 [notice] 10112#10112: signal process started
2018/06/29 09:09:34 [error] 10112#10112: invalid PID number "" in "/usr/local/openresty/nginx/logs/nginx.pid"
2018/06/29 09:14:52 [notice] 10206#10206: signal process started
2018/06/29 09:14:58 [notice] 10209#10209: signal process started
2018/06/29 09:30:18 [notice] 10444#10444: signal process started
2018/06/29 09:32:03 [notice] 10470#10470: signal process started
2018/06/29 09:44:49 [notice] 10614#10614: signal process started
2018/06/29 09:44:49 [alert] 10614#10614: kill(10009, 1) failed (3: No such process)
2018/06/29 09:45:05 [notice] 10616#10616: signal process started
2018/06/29 09:45:05 [alert] 10616#10616: kill(10009, 1) failed (3: No such process)
2018/06/29 09:50:38 [notice] 10681#10681: signal process started
2018/06/29 09:50:38 [alert] 10681#10681: kill(10009, 1) failed (3: No such process)