技术解析

深夜处理 bug:重启 openresty(nginx)后, 403 forbidden,连服务器 ip 都无法访问了
0
2021-05-19 22:31:56
idczone

手贱重启了 openresty,突然站点就 403 forbidden。搞了一天了,真心累,请教同学们。

配置环境:centos7+openresty(nginx)+php+mysql systemctl status openresty ,systemctl status php-fpm,systemctl status mysql,都是 active status,绿色。

lsof -i :80

COMMAND PID USER FD TYPE DEVICE SIZE/OFF NODE NAME

nginx 32255 root 6u IPv4 170338 0t0 TCP localhost:http (LISTEN)

nginx 32332 root 6u IPv4 170338 0t0 TCP localhost:http (LISTEN)

netstat -an |grep 80

tcp 0 0 127.0.0.1:80 0.0.0.0:* LISTEN

unix 2 [ ACC ] STREAM LISTENING 28071 /var/lib/mysql/mysql.sock

unix 3 [ ] STREAM CONNECTED 11800

unix 3 [ ] STREAM CONNECTED 11801 /run/systemd/journal/stdout

ps aux | gre美国服务器p php

root 19363 0.0 3.1 470292 31572 ? Ss 00:20 0:04 php-fpm: master process (/etc/ph-fpm.conf)

nginx 19364 0.0 0.7 470292 7228 ? S 00:20 0:00 php-fpm: pool www

nginx 19365 0.0 0.7 470292 7228 ? S 00:20 0:00 php-fpm: pool www

nginx 19366 0.0 0.7 470292 7228 ? S 00:20 0:00 php-fpm: pool www

nginx 19367 0.0 0.7 470292 7228 ? S 00:20 0:00 php-fpm: pool www

nginx 19368 0.0 0.7 470292 7232 ? S 00:20 0:00 php-fpm: pool www

nginx 19369 0.0 0.7 470292 7232 ? S 00:20 0:00 php-fpm: pool www

nginx 19370 0.0 0.7 470292 7232 ? S 00:20 0:00 php-fpm: pool www

nginx 19371 0.0 0.7 470292 7232 ? S 00:20 0:00 php-fpm: pool www

root 32204 0.0 0.0 112704 968 pts/0 R+ 17:57 0:00 grep --color=auto php

netstat -plant

Active Internet connections (servers and established)

Proto Recv-Q Send-Q Local Address Foreign Address State PID/Program name

tcp 0 0 127.0.0.1:80 0.0.0.0:* LISTEN 32255/nginx: master

tcp 0 0 0.0.0.0:22 0.0.0.0:* LISTEN 730/sshd

tcp 0 0 45.79.66.57:22 113.118.253.226:4315 ESTABLISHED 32434/sshd: [email protected]

tcp 0 36 45.79.66.57:22 113.118.253.226:4860 ESTABLISHED 32584/sshd: [email protected]

tcp 0 0 45.79.66.57:22 113.118.253.226:3505 ESTABLISHED 32115/sshd: [email protected]

tcp6 0 0 :::9000 :::* LISTEN 19363/php-fpm: mast

tcp6 0 0 :::3306 :::* LISTEN 3299/mysqld

tcp6 0 0 :::22 :::* LISTEN 730/sshd

/usr/local/openresty/nginx/conf/nginx.conf

user root;

worker_processes 1;

#error_log logs/error.log;

#error_log logs/error.log notice;

#error_log logs/error.log info;

#pid logs/nginx.pid;

events { worker_connections 1024; }

http { include mime.types; default_type application/octet-stream;

#log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
#                  '$status $body_bytes_sent "$http_referer" '
#                  '"$http_user_agent" "$http_x_forwarded_for"';

#access_log  logs/access.log  main;

sendfile        on;
#tcp_nopush     on;

#keepalive_timeout  0;
keepalive_timeout  65;

client_max_body_size 8m;    #允许客户端请求的最大单文件字节数
client_body_buffer_size 2m;  #缓冲区代理缓冲用户端请求的最大字节

fastcgi_buffers 8 16k;
fastcgi_buffer_size 32k;
fastcgi_connect_timeout 300;
fastcgi_send_timeout 300;
fastcgi_read_timeout 300;

gzip  on;
#WAF
lua_shared_dict limit 50m;
lua_shared_dict guard_dict 100m;
lua_shared_dict dict_captcha 70m;
lua_package_path "/usr/local/openresty/nginx/conf/waf/?.lua";
init_by_lua_file "/usr/local/openresty/nginx/conf/waf/init.lua";
access_by_lua_file "/usr/local/openresty/nginx/conf/waf/access.lua";

server {
    listen       127.0.0.1;
    server_name  kiwikiwifly.com;

    #charset koi8-r;

    #access_log  logs/host.access.log  main;

    location / {
        root   html;
        index  index.php index.html index.htm;
    }

    #error_page  404              /404.html;

    # redirect server error pages to the static page /50x.html
    #
    error_page   500 502 503 504  /50x.html;
    location = /50x.html {
        root   html;
    }

    # proxy the PHP scripts to Apache listening on 127.0.0.1:80
    #
    #location ~ \.php$ {
    #    proxy_pass   http://127.0.0.1;
    #}

    # pass the PHP scripts to FastCGI server listening on 127.0.0.1:9000
    #
    
    location ~ \.php$ {
        root           html;
        fastcgi_pass   127.0.0.1:9000;
        fastcgi_index  index.php;
        fastcgi_param  SCRIPT_FILENAME  $document_root$fastcgi_script_name;
        include        fastcgi_params;
    }

    # deny access to .htaccess files, if Apache's document root
    # concurs with nginx's one
    #
    #location ~ /\.ht {
    #    deny  all;
    #}
}


# another virtual host using mix of IP-, name-, and port-based configuration
#
#server {
#    listen       8000;
#    listen       somename:8080;
#    server_name  somename  alias  another.alias;

#    location / {
#        root   html;
#        index  index.html index.htm;
#    }
#}


# HTTPS server
#
#server {
#    listen       443 ssl;
#    server_name  localhost;

#    ssl_certificate      cert.pem;
#    ssl_certificate_key  cert.key;

#    ssl_session_cache    shared:SSL:1m;
#    ssl_session_timeout  5m;

#    ssl_ciphers  HIGH:!aNULL:!MD5;
#    ssl_prefer_server_ciphers  on;

#    location / {
#        root   html;
#        index  index.html index.htm;
#    }
#}

}

and /usr/local/openresty/nginx/logs/error.log

2018/06/26 21:27:10 [alert] 740#740: *7927 open socket #10 left in connection 12

2018/06/26 21:27:10 [alert] 740#740: *7926 open socket #3 left in connection 13

2018/06/26 21:27:10 [alert] 740#740: aborting

只有到 26 号 error.log


1. 给 server 那段单独写一个 error_log
2. 试试把 WAF 相关的配置先注释掉

2.已经注释了,1 给 server 那段单独写一个 error_log ?请问这个如何操作?

看一下 nginx 有没有 web 根路径的读权限,如果有,在根路径下创建一个 xxxx.php 文件,里面随便输出点字符串,然后访问,看能否访问,能,说明你站点 php 逻辑问题,不能,nginx 问题,通过日志把问题,还是找不到,直接上 strace

livid 的办法是正解。
开 error_log,debug 级别。就能看到为啥出错了。

你这边怎么又是 403,另一个帖子 502。listen 127.0.0.1,你是直接 在本地 curl 127.0.0.1 出现 403/502 ?
如果是访问你的站点的公网地址,前面那一层怎么访问 127.0.0.1 ?

user nobody nobody;
worker_processes 2;
error_log /var/log/nginx/error.log debug ; 这是错误日志的路径,会了么?
pid /var/run/nginx/nginx.pid;

Sorry,Sorry,是 502 bad gateway 的提示。


你好,按照您的建议,把你提供的这个写在了 http 段外面,重新 reload openresty 成功,error_log /var/log/nginx/error.log 内容:
2018/06/29 09:06:08 [notice] 10008using the "epoll" event method
2018/06/29 09:06:08 [notice] 10008openresty/1.13.6.2
2018/06/29 09:06:08 [notice] 10008built by gcc 4.8.5 20150623 (Red Hat 4.8.5-16) (GCC)
2018/06/29 09:06:08 [notice] 10008OS: Linux 3.10.0-862.3.2.el7.x86_64
2018/06/29 09:06:08 [notice] 10008getrlimit(RLIMIT_NOFILE): 1024:4096
2018/06/29 09:06:08 [notice] 10009start worker processes
2018/06/29 09:06:08 [notice] 10009start worker process 10011
2018/06/29 09:14:52 [notice] 10009signal 1 (SIGHUP) received from 10206, reconfiguring
2018/06/29 09:14:52 [notice] 10009reconfiguring
2018/06/29 09:14:52 [notice] 10009using the "epoll" event method
2018/06/29 09:14:52 [notice] 10009start worker processes
2018/06/29 09:14:52 [notice] 10009start worker process 10207
2018/06/29 09:14:52 [notice] 10011gracefully shutting down
2018/06/29 09:14:52 [notice] 10011exiting
2018/06/29 09:14:52 [notice] 10011exit
2018/06/29 09:14:52 [notice] 10009signal 17 (SIGCHLD) received from 10011
2018/06/29 09:14:52 [notice] 10009worker process 10011 exited with code 0
2018/06/29 09:14:52 [notice] 10009signal 29 (SIGIO) received
2018/06/29 09:14:58 [notice] 10009signal 1 (SIGHUP) received from 10209, reconfiguring
2018/06/29 09:14:58 [notice] 10009reconfiguring
2018/06/29 09:14:58 [notice] 10009using the "epoll" event method
2018/06/29 09:14:58 [notice] 10009start worker processes
2018/06/29 09:14:58 [notice] 10009start worker process 10210
2018/06/29 09:14:58 [notice] 10207gracefully shutting down
2018/06/29 09:14:58 [notice] 10207exiting
2018/06/29 09:14:58 [notice] 10207exit
2018/06/29 09:14:58 [notice] 10009signal 17 (SIGCHLD) received from 10207
2018/06/29 09:14:58 [notice] 10009worker process 10207 exited with code 0
2018/06/29 09:14:58 [notice] 10009signal 29 (SIGIO) received


/var/run/nginx/nginx.pid 仅仅输出了
10009
这个提示。请问我应该修改什么地方呢?服务器 ip 还是无法访问

最新的 /usr/local/openresty/nginx/log/error.log

8/06/28 18:04:18 [notice] 32331signal process started
2018/06/28 22:19:32 [emerg] 2711bind() to 127.0.0.1:80 failed (98: Address already in use)
2018/06/28 22:19:32 [emerg] 2711bind() to 127.0.0.1:80 failed (98: Address already in use)
2018/06/28 22:19:32 [emerg] 2711bind() to 127.0.0.1:80 failed (98: Address already in use)
2018/06/28 22:19:32 [emerg] 2711bind() to 127.0.0.1:80 failed (98: Address already in use)
2018/06/28 22:19:32 [emerg] 2711bind() to 127.0.0.1:80 failed (98: Address already in use)
2018/06/28 22:19:32 [emerg] 2711still could not bind()
2018/06/29 08:38:40 [emerg] 9553open() "/var/run/nginx/nginx.pid" failed (2: No such file or directory)
2018/06/29 08:39:35 [emerg] 9589open() "/var/run/nginx/nginx.pid" failed (2: No such file or directory)
2018/06/29 08:51:43 [emerg] 9757open() "/var/log/nginx/error.log" failed (2: No such file or directory)
2018/06/29 08:55:53 [notice] 9801signal process started
2018/06/29 08:55:53 [error] 9801invalid PID number "" in "/var/run/nginx/nginx.pid"
2018/06/29 09:09:34 [notice] 10112signal process started
2018/06/29 09:09:34 [error] 10112invalid PID number "" in "/usr/local/openresty/nginx/logs/nginx.pid"
2018/06/29 09:14:52 [notice] 10206signal process started
2018/06/29 09:14:58 [notice] 10209signal process started
2018/06/29 09:30:18 [notice] 10444signal process started
2018/06/29 09:32:03 [notice] 10470signal process started
2018/06/29 09:44:49 [notice] 10614signal process started
2018/06/29 09:44:49 [alert] 10614kill(10009, 1) failed (3: No such process)
2018/06/29 09:45:05 [notice] 10616signal process started
2018/06/29 09:45:05 [alert] 10616kill(10009, 1) failed (3: No such process)
2018/06/29 09:50:38 [notice] 10681signal process started
2018/06/29 09:50:38 [alert] 10681kill(10009, 1) failed (3: No such process)

数据地带为您的网站提供全球顶级IDC资源
在线咨询
专属客服