I have a Node.JS Express API (MySQL) hosted on Nginx behind Cloudflare (2 instances running). I'm getting a lot of 504 timeout on Roblox and upstream timed out on Nginx. I have never seen a request I sent with Postman fail. I think it happens more under load. These instances are processing processing 11M requests a week. This is hosted on a 16 core, 64 GB RAM, dedicated server with 2-3 load average
Nginx error log spams these:
upstream timed out (110: Connection timed out) while reading response header from upstream
no live upstreams while connecting to upstream
upstream prematurely closed connection while reading response header from upstream
The upstream timed out errors are the concern as they are the majority of the errors.
Generally, I don't do too much processing on the API. I have less then a dozen endpoints that are mostly simple DB selects.
Can someone direct me in the right area to resolve this? Is it my Nginx config, do I need more instances, is it my design, is it Roblox, is it Cloudflare? I read Node.js can handle this (under one instance), so I tried to adjust worker connections in Nginx which caused more no live upstream errors. I cannot wrap my head around what the bottle neck is.
Site Config
proxy_cache_path /tmp/NGINX_cache/ keys_zone=backcache:10m;
map $http_upgrade $connection_upgrade {
default upgrade;
' ' close;
}
upstream nodejs {
# Use IP Hash for session persistence
ip_hash;
keepalive 90;
# List of Node.js application servers
server localhost:9000;
server localhost:9001;
}
# HTTP: www and non-www to https
server {
listen 80;
listen [::]:80;
server_name example.com www.example.com;
return 301 https://example.com$request_uri;
}
# HTTPS: non-www to www
server {
listen 443 ssl;
listen [::]:443 ssl;
ssl_certificate /example/example.com.cert.pem;
ssl_certificate_key /example/example.com.key.pem;
server_name example.com;
return 301 https://www.example.com$request_uri;
}
# HTTPS: www
server {
listen 443 ssl http2;
listen [::]:443 ssl http2;
ssl_certificate /example/example.com.cert.pem;
ssl_certificate_key /example/example.com.key.pem;
server_name www.example.com;
location / {
return 301 $scheme://www.example.example$request_uri;
}
location /api {
proxy_pass https://nodejs;
proxy_cache backcache;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection 'upgrade';
proxy_set_header Host $host;
proxy_cache_bypass $http_upgrade;
proxy_read_timeout 90;
proxy_redirect https://nodejs https://www.example.com;
}
location /api_staging {
proxy_pass https://localhost:8000;
proxy_cache backcache;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection 'upgrade';
proxy_set_header Host $host;
proxy_cache_bypass $http_upgrade;
proxy_read_timeout 90;
proxy_redirect https://localhost:8000 https://www.example.com;
}
location /api_development {
proxy_pass https://localhost:7000;
proxy_cache backcache;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection 'upgrade';
proxy_set_header Host $host;
proxy_cache_bypass $http_upgrade;
proxy_read_timeout 90;
proxy_redirect https://localhost:7000 https://www.example.com;
}
}
Nginx Config
user www-data;
worker_processes auto;
pid /run/nginx.pid;
include /etc/nginx/modules-enabled/*.conf;
events {
worker_connections 1000;
# multi_accept on;
}
http {
##
# Basic Settings
##
sendfile on;
tcp_nopush on;
tcp_nodelay on;
keepalive_timeout 65;
types_hash_max_size 2048;
# server_tokens off;
# server_names_hash_bucket_size 64;
# server_name_in_redirect off;
include /etc/nginx/mime.types;
default_type application/octet-stream;
##
# SSL Settings
##
ssl_protocols TLSv1 TLSv1.1 TLSv1.2 TLSv1.3; # Dropping SSLv3, ref: POODLE
ssl_prefer_server_ciphers on;
##
# Logging Settings
##
access_log /var/log/nginx/access.log;
error_log /var/log/nginx/error.log;
##
# Gzip Settings
##
gzip on;
# gzip_vary on;
# gzip_proxied any;
# gzip_comp_level 6;
# gzip_buffers 16 8k;
# gzip_http_version 1.1;
# gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript;
##
# Virtual Host Configs
##
include /etc/nginx/conf.d/*.conf;
include /etc/nginx/sites-enabled/*;
client_max_body_size 100M;
}
#mail {
# # See sample authentication script at:
# # http://wiki.nginx.org/ImapAuthenticateWithApachePhpScript
#
# # auth_http localhost/auth.php;
# # pop3_capabilities "TOP" "USER";
# # imap_capabilities "IMAP4rev1" "UIDPLUS";
#
# server {
# listen localhost:110;
# protocol pop3;
# proxy on;
# }
#
# server {
# listen localhost:143;
# protocol imap;
# proxy on;
# }
#}
Cloudflare Edits
Proxied is on
Full strict SSL
All Roblox IPs are allowed through firewall