Nginx reverse proxy for Nodejs server SSL_ERROR_RX_RECORD_TOO_LONG

I’m using AWS Beanstalk to configure a multicontainer docker environment to both serve my PHP Docker Application parallel to my NodeJS Server, running on port 3000.

I have a running Express Server, listening on port 3000. I now want to be able to call my NodeJS server at https://nodejs.my-domain.com:3000. Nginx should now terminate the SSL connection and forward all traffix to my NodeJS Express server.

So far I can successfully reach my PHP Application both with and without https. I can also reach my NodeJS Application without SSL at http://nodejs.my-domain.com:3000. But as soon as I call it with https I get the Broser Error SSL_ERROR_RX_RECORD_TOO_LONG.

Nginx config file looks like this:

log_format healthd '$msec"$uri"'
          '$status"$request_time"$upstream_response_time"'
          '$http_x_forwarded_for';

upstream nodejs {
    server 127.0.0.1:3000;
    keepalive 256;
}

server {
    listen 80;
    listen [::]:80;
    listen 443 ssl;
    listen [::]:443 ssl;

    server_name nodejs.my-domain.com

    ssl_certificate /etc/nginx/certs/nginx-selfsigned.crt;
    ssl_certificate_key /etc/nginx/certs/nginx-selfsigned.key;
    ssl_session_timeout 1d;
    ssl_session_cache shared:MozSSL:10m;
    ssl_protocols TLSv1.2 TLSv1.3;
    ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384;
    ssl_prefer_server_ciphers off;

    if ($time_iso8601 ~ "^(d{4})-(d{2})-(d{2})T(d{2})") {
        set $year $1;
        set $month $2;
        set $day $3;
        set $hour $4;
    }

    access_log /var/log/nginx/access.log main;
    access_log /var/log/nginx/healthd/application.log.$year-$month-$day-$hour healthd;

    location / {
        proxy_http_version 1.1;
        proxy_set_header Upgrade $http_upgrade;
        proxy_set_header Connection "upgrade";
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header Host $http_host;
        proxy_set_header X-NginX-Proxy true;

        proxy_pass http://nodejs;
        proxy_redirect off;
    }
}

server {
    listen 80;
    listen [::]:80;
    listen 443 ssl http2 default_server;
    listen [::]:443 ssl http2 default_server;

    server_name localhost my-domain.com;
    root /var/www/public;

    ssl_certificate /etc/nginx/certs/nginx-selfsigned.crt;
    ssl_certificate_key /etc/nginx/certs/nginx-selfsigned.key;
    ssl_session_timeout 1d;
    ssl_session_cache shared:MozSSL:10m;
    ssl_protocols TLSv1.2 TLSv1.3;
    ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384;
    ssl_prefer_server_ciphers off;

    if ($time_iso8601 ~ "^(d{4})-(d{2})-(d{2})T(d{2})") {
        set $year $1;
        set $month $2;
        set $day $3;
        set $hour $4;
    }

    access_log /var/log/nginx/access.log main;
    access_log /var/log/nginx/healthd/application.log.$year-$month-$day-$hour healthd;

    index index.php index.html index.htm;

    if ($ssl_protocol = "") {
        rewrite ^ https://$host$request_uri? permanent;
    }

    location / {
        try_files $uri $uri/ /index.php?$args;
    }

    location ~ [^/].php(/|$) {
        try_files $uri =404;
        fastcgi_split_path_info ^(.+?.php)(/.*)$;
        include fastcgi_params;
        fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
        fastcgi_param PATH_INFO $fastcgi_path_info;
        fastcgi_param PATH_TRANSLATED $document_root$fastcgi_path_info;

        fastcgi_pass php:9000;
        fastcgi_index index.php;
    }
}

I’m not sure which logs I have to pay attention to. Here is some output of various log files:

My access.log looks like this:

XX.X.XXX.X - - [18/Mar/2020:12:12:12 +0000] "GET / HTTP/1.1" 502 559 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36" "-"
XXX.XX.XX.XX - - [18/Mar/2020:12:27:09 +0000] "GET / HTTP/1.1" 502 157 "-" "Mozilla/5.0 zgrab/0.x" "-"
XX.XXX.XX.XXX - - [18/Mar/2020:12:54:16 +0000] "GET http://example.com/ HTTP/1.1" 502 157 "-" "AWS Security Scanner" "-"
XX.XXX.XX.XXX - - [18/Mar/2020:12:54:17 +0000] "GET http://XXX.XXX.XXX.XXX/ HTTP/1.1" 502 157 "-" "AWS Security Scanner" "-"
XX.XXX.XX.XXX - - [18/Mar/2020:12:54:17 +0000] "GET http://[::XXXX:XXXXX:XXXXX]/ HTTP/1.1" 502 157 "-" "AWS Security Scanner" "-"
XX.XXX.XX.XXX - - [18/Mar/2020:12:54:17 +0000] "GET http://XXX.XXX.XXX.XXX/latest/dynamic/instance-identity/document HTTP/1.1" 502 157 "-" "AWS Security Scanner" "-"
XX.XXX.XX.XXX - - [18/Mar/2020:12:54:18 +0000] "GET http://[::XXXX:XXXXX:XXXXX]/latest/dynamic/instance-identity/document HTTP/1.1" 502 157 "-" "AWS Security Scanner" "-"
XX.XXX.XX.XXX - - [18/Mar/2020:12:54:18 +0000] "GET / HTTP/1.1" 502 157 "-" "AWS Security Scanner" "-"
XX.XXX.XX.XXX - - [18/Mar/2020:12:54:18 +0000] "GET / HTTP/1.1" 502 157 "-" "AWS Security Scanner" "-"
XX.XXX.XX.XXX - - [18/Mar/2020:12:54:19 +0000] "GET / HTTP/1.1" 502 157 "-" "AWS Security Scanner" "-"
XX.XXX.XX.XXX - - [18/Mar/2020:12:54:19 +0000] "GET /latest/dynamic/instance-identity/document HTTP/1.1" 502 157 "-" "AWS Security Scanner" "-"
XX.XXX.XX.XXX - - [18/Mar/2020:12:54:19 +0000] "GET /latest/dynamic/instance-identity/document HTTP/1.1" 502 157 "-" "AWS Security Scanner" "-"
XX.XXX.XX.XXX - - [18/Mar/2020:12:54:20 +0000] "CONNECT X.XXX.XXX.XXX:80 HTTP/1.0" 400 157 "-" "-" "-"
XX.XXX.XX.XXX - - [18/Mar/2020:12:54:20 +0000] "CONNECT X.XXX.XXX.XXX:80 HTTP/1.0" 400 157 "-" "-" "-"
XX.XXX.XX.XXX - - [18/Mar/2020:12:54:20 +0000] "CONNECT X.XXX.XXX.XXX:80 HTTP/1.0" 400 157 "-" "-" "-"
XX.XXX.XX.XXX - - [18/Mar/2020:12:54:21 +0000] "CONNECT X.XXX.XXX.XXX:80 HTTP/1.0" 400 157 "-" "-" "-"
XX.XXX.XX.XXX - - [18/Mar/2020:12:54:21 +0000] "CONNECT X.XXX.XXX.XXX:80 HTTP/1.0" 400 157 "-" "-" "-"
XX.XXX.XX.XXX - - [18/Mar/2020:12:54:21 +0000] "x16x03x01x00xD2x01x00x00xCEx03x03x11xB9xBBxFDxF6axD4xAFQx1FxC0x99jxFA#xBCXxF9A}'xC9x00xF9x98K0x88xBAxEAxC0x09x00x00bxC00xC0,xC0/xC0+x00x9Fx00x9ExC02xC0.xC01xC0-x00xA5x00xA1x00xA4x00xA0xC0(xC0$xC0x14xC0" 400 157 "-" "-" "-"
XX.XXX.XX.XXX - - [18/Mar/2020:12:54:22 +0000] "x16x03x01x00xD2x01x00x00xCEx03x03xD7xEDxA5|xF8uxCAx1CxD17rx8B1xD5x8FxD07x9CxD7Yx06h" 400 157 "-" "-" "-"
XX.XXX.XX.XXX - - [18/Mar/2020:12:54:22 +0000] "x16x03x01x00xD2x01x00x00xCEx03x033':xC6xE6x90xA8M" 400 157 "-" "-" "-"
XX.XXX.XX.XXX - - [18/Mar/2020:12:54:22 +0000] "x16x03x01x00xD2x01x00x00xCEx03x03" 400 157 "-" "-" "-"
XX.XXX.XX.XXX - - [18/Mar/2020:12:54:23 +0000] "x16x03x01x00xD2x01x00x00xCEx03x03xCB=xFAixFAx8Fx08x1Ex98xCEcx19x18xDDxA0xAExC4{x18ExFDxC2zxC3x97xB5x97xFEWxC0xA6~x00x00bxC00xC0,xC0/xC0+x00x9Fx00x9ExC02xC0.xC01xC0-x00xA5x00xA1x00xA4x00xA0xC0(xC0$xC0x14xC0" 400 157 "-" "-" "-"

And in my error.log I found the following:

2020/03/18 11:01:40 [warn] 1#1: "ssl_stapling" ignored, issuer certificate not found for certificate "/etc/nginx/certs/nginx-selfsigned.crt"
2020/03/18 11:01:40 [warn] 1#1: "ssl_stapling" ignored, issuer certificate not found for certificate "/etc/nginx/certs/nginx-selfsigned.crt"
2020/03/18 11:14:44 [warn] 1#1: "ssl_stapling" ignored, issuer certificate not found for certificate "/etc/nginx/certs/nginx-selfsigned.crt"
2020/03/18 11:14:44 [warn] 1#1: "ssl_stapling" ignored, issuer certificate not found for certificate "/etc/nginx/certs/nginx-selfsigned.crt"

It also looks like there is something wrong with my healthd loggin config. There are many entries like this in my healthd/deamon.log:

# Logfile created on 2020-03-17 20:33:13 +0000 by logger.rb/47272
A, [2020-03-17T20:33:14.155980 #2972]   ANY -- : healthd daemon 1.0.3 initialized
W, [2020-03-17T20:33:14.249690 #2972]  WARN -- : log file "/var/log/nginx/healthd/application.log.2020-03-17-20" does not exist
W, [2020-03-17T20:33:14.249690 #2972]  WARN -- : log file "/var/log/nginx/healthd/application.log.2020-03-17-20" does not exist
[...]
A, [2020-03-17T20:34:03.782734 #4025]   ANY -- : healthd daemon 1.0.3 initialized
W, [2020-03-17T20:34:03.858118 #4025]  WARN -- : log file "/var/log/containers/nginx-proxy/healthd/application.log.2020-03-17-20" does not exist
W, [2020-03-17T20:34:03.858118 #4025]  WARN -- : log file "/var/log/containers/nginx-proxy/healthd/application.log.2020-03-17-20" does not exist
[...]

Answer

I found a solution to my problem. There were two things I was missing:

1. I am using a multi container docker environment with Nginx and Nodejs both running within their own container. In order for Nginx to be able to reach my Nodejs server I had to create a link between Nginx and Nodejs. But in my configuration file I configured the link within the Nodejs section instead of the Nginx section. I now added "links": ["node"] to the Nginx section within my Dockerrun.aws.conf which now looks like this:

{
    "AWSEBDockerrunVersion": 2,
    "containerDefinitions": [
        {
            "name": "nginx-proxy",
            "image": "MY_NGINX_IMAGE_ON_PRIVATE_REGISTRY",
            "portMappings": [
                {
                    "containerPort": 80,
                    "hostPort": 80
                },
                {
                    "containerPort": 443,
                    "hostPort": 443
                }
            ],
            "links": ["node"]
            [...]
        },
        {
            "name": "node",
            "image": "MY_NODEJS_IMAGE_ON_PRIVATE_REGISTRY",
            "portMappings": [
                {
                    "containerPort": 3000,
                    "hostPort": 3000
                }
            ]
        }
        [...]
    ]
    [...]
}

2. Further more instead of setting the upstream within my Nginx config to 127.0.0.1:3000 I had to do the upstream to my Nodejs docker container, which I named node: server node:3000. So my /var/nginx/conf.d/default.conf now looks like this:

[...]

upstream nodejs {
    server node:3000;
    keepalive 256;
}

server {
    listen 443 ssl;

    server_name websockets.my-domain.com;

    ssl_certificate /etc/letsencrypt/live/websockets.my-domain.com/fullchain.pem;
    ssl_certificate_key /etc/letsencrypt/live/websockets.my-domain.com/privkey.pem;

    [...]

    location / {
        proxy_http_version 1.1;
        proxy_set_header Upgrade ${DOLLAR}http_upgrade;
        proxy_set_header Connection "upgrade";
        proxy_set_header X-Real-IP ${DOLLAR}remote_addr;
        proxy_set_header X-Forwarded-For ${DOLLAR}proxy_add_x_forwarded_for;
        proxy_set_header Host ${DOLLAR}http_host;
        proxy_set_header X-NginX-Proxy true;

        proxy_pass http://nodejs;
        proxy_redirect off;
    }
}

[...]