How to make Nginx access logs more readable with json

First we need to add a log format directive in the http context. We’ll call this log format json_logs and use the json escape parameter(Nginx version 1.11.8). Example:

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
http {
	# ...

	log_format json_logs escape=json '{'
		'"time_local":"$time_local",'
		'"remote_addr": "$remote_addr",'                            # client IP
		'"request":"$request",'                                     # full path no arguments if the request
		'"status": "$status",'                                      # response status code
		'"request_time":"$request_time",'                           # request processing time in seconds with msec resolution
		'"remote_user": "$remote_user",'                            # client HTTP username
		'"remote_port": "$remote_port",'                            # client port
		'"body_bytes_sent": "$body_bytes_sent",'                    # the number of body bytes exclude headers sent to a client
		'"bytes_sent": "$bytes_sent", '                             # the number of bytes sent to a client
		'"request_length": "$request_length", '                     # request length (including headers and body)
		'"connection_requests": "$connection_requests",'            # number of requests made in connection
		'"http_host": "$http_host", '                               # the request Host: header
		'"http_referrer":"$http_referer",'
		'"upstream": "$upstream_addr", '                            # upstream backend server for proxied requests
		'"upstream_connect_time": "$upstream_connect_time", '       # upstream handshake time incl. TLS
		'"upstream_header_time": "$upstream_header_time", '         # time spent receiving upstream headers
		'"upstream_response_time": "$upstream_response_time", '     # time spend receiving upstream body
		'"upstream_response_length": "$upstream_response_length", ' # upstream response length
		'"upstream_cache_status": "$upstream_cache_status", '       # cache HIT/MISS where applicable
		'"ssl_protocol": "$ssl_protocol", '                         # TLS protocol
		'"ssl_cipher": "$ssl_cipher", '                             # TLS cipher
		'"scheme": "$scheme", '                                     # http or https
		'"http_user_agent":"$http_user_agent"'
	'}';

	# ...
}

After that we just need to reference the json_logs format declared before inside our access log, like so:

1
2
3
4
5
6
7
8
9
server{
	
	# ...

	access_log /logs/nginx-access.json json_logs;
	error_log /logs/nginx-error.log;
	
	# ....
}

New Nginx logs now:

1
2
3
4
{"time_local":"18/Feb/2021:18:26:49 +0000","remote_addr": "127.0.0.1","request":"GET /login HTTP/1.1","status": "200","request_time":"0.004","remote_user": "","remote_port": "58292","body_bytes_sent": "1076","bytes_sent": "1240", "request_length": "751", "connection_requests": "1","http_host": "localhost", "http_referrer":"http://localhost/dashboard","upstream": "[::1]:5000, 127.0.0.1:5000", "upstream_connect_time": "0.000, 0.000", "upstream_header_time": "-, 0.004", "upstream_response_time": "0.000, 0.004", "upstream_response_length": "0, 1076", "upstream_cache_status": "", "ssl_protocol": "", "ssl_cipher": "", "scheme": "http", "http_user_agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 11_2_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.150 Safari/537.36"}
{"time_local":"18/Feb/2021:18:26:52 +0000","remote_addr": "127.0.0.1","request":"POST /api/request_login_link HTTP/1.1","status": "200","request_time":"0.870","remote_user": "","remote_port": "58292","body_bytes_sent": "16","bytes_sent": "178", "request_length": "862", "connection_requests": "2","http_host": "localhost", "http_referrer":"http://localhost/login","upstream": "127.0.0.1:5000", "upstream_connect_time": "0.001", "upstream_header_time": "0.870", "upstream_response_time": "0.870", "upstream_response_length": "16", "upstream_cache_status": "", "ssl_protocol": "", "ssl_cipher": "", "scheme": "http", "http_user_agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 11_2_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.150 Safari/537.36"}
{"time_local":"18/Feb/2021:18:27:04 +0000","remote_addr": "127.0.0.1","request":"GET /login?login_token=B3SIA4J-glLQHd-MJaYFWsXoH4ILST-vIsvmmnPfOKLMQEQCbG8ehLnC7vl-4c-BqoU&user_email=potato%40gmail.com HTTP/1.1","status": "302","request_time":"0.008","remote_user": "","remote_port": "58292","body_bytes_sent": "227","bytes_sent": "762", "request_length": "777", "connection_requests": "3","http_host": "localhost", "http_referrer":"","upstream": "127.0.0.1:5000", "upstream_connect_time": "0.000", "upstream_header_time": "0.008", "upstream_response_time": "0.008", "upstream_response_length": "227", "upstream_cache_status": "", "ssl_protocol": "", "ssl_cipher": "", "scheme": "http", "http_user_agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 11_2_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.150 Safari/537.36"}
{"time_local":"18/Feb/2021:18:27:04 +0000","remote_addr": "127.0.0.1","request":"GET /dashboard HTTP/1.1","status": "200","request_time":"0.010","remote_user": "","remote_port": "58292","body_bytes_sent": "26844","bytes_sent": "27340", "request_length": "924", "connection_requests": "4","http_host": "localhost", "http_referrer":"","upstream": "127.0.0.1:5000", "upstream_connect_time": "0.001", "upstream_header_time": "0.010", "upstream_response_time": "0.010", "upstream_response_length": "26844", "upstream_cache_status": "", "ssl_protocol": "", "ssl_cipher": "", "scheme": "http", "http_user_agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 11_2_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.150 Safari/537.36"}

A closer look:

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
{
  "time_local": "18/Feb/2021:18:27:09 +0000",
  "remote_addr": "127.0.0.1",
  "request": "GET /dashboard/mailboxes/340d8f4c-c056-4b62-bf4f-0d537790cddd HTTP/1.1",
  "status": "200",
  "request_time": "0.006",
  "remote_user": "",
  "remote_port": "58292",
  "body_bytes_sent": "3937",
  "bytes_sent": "4432",
  "request_length": "1058",
  "connection_requests": "7",
  "http_host": "localhost",
  "http_referrer": "http://localhost/dashboard",
  "upstream": "127.0.0.1:5000",
  "upstream_connect_time": "0.001",
  "upstream_header_time": "0.006",
  "upstream_response_time": "0.006",
  "upstream_response_length": "3937",
  "upstream_cache_status": "",
  "ssl_protocol": "",
  "ssl_cipher": "",
  "scheme": "http",
  "http_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 11_2_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.150 Safari/537.36"
}