Nginx日志统计展示
2022/5/6 7:13:41
本文主要是介绍Nginx日志统计展示，对大家解决编程问题具有一定的参考价值，需要的程序猿们随着小编来一起学习吧！
一、 效果展示
二、流程
三、 Nginx 配置
nginx.conf，修改日志格式，指定需要收集的字段及字段分割
http{
........
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent"';
........
}
四、 Filebeat 配置
修改filebeat.yml
egrep -v "^#|^ #|^$" filebeat.yml
filebeat.inputs:
- type: log
enabled: true
paths:
- /var/log/nginx/xxx.log
fields:
file_type: nginx-log
fields_under_root: true
encoding: utf-8
filebeat.config.modules:
path: ${path.config}/modules.d/*.yml
reload.enabled: false
setup.template.settings:
index.number_of_shards: 1
setup.kibana:
output.logstash:
hosts: ["xxx.xxx.xxx.xxx:xxxxx"]
processors:
- add_host_metadata: ~
- add_cloud_metadata: ~
- add_docker_metadata: ~
- add_kubernetes_metadata: ~
五、 Logstash 配置
安装geoip插件
./bin/logstash-plugin install logstash-filter-geoip
修改logstash.conf
input {
beats {
host => '0.0.0.0'
port => xxxxx
codec => plain{ charset => "GBK" }
}
}
filter {
if [file_type] == "nginx-log"{
grok {
match => [
'message', '%{IPORHOST:remote_addr} - %{NOTSPACE:remote_user} \[%{HTTPDATE:time_local}\] \"%{WORD:method} %{NOTSPACE:request_uri} %{URIPROTO:proto}/%{NUMBER:httpversion}\" %{NUMBER:status} (?:%{NUMBER:size}|-) %{QS:referrer} %{QS:user_agent}'
]
}
geoip {
source => "remote_addr"
fields => ["location","country_name","city_name","region_name"]
}
mutate {
remove_field => ["host", "agent"]
}
}
}
output {
if [file_type] == "nginx-log"{
elasticsearch {
hosts => ["xxx.xxx.xxx.xxx:xxxxx"]
index => "%{file_type}-%{+YYYY.MM.dd}"
user => 'xxxx'
password => 'xxxxxx'
}
}
}
六、 Elasticsearch 配置
没有需要特别配置的，保证运行正常就行
七、 Grafana 添加数据源
八、 Json导出（只展示panels）
"panels": [
{
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 4,
"x": 0,
"y": 0
},
"id": 12,
"options": {
"colorMode": "none",
"graphMode": "none",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"last"
],
"fields": "",
"values": false
},
"textMode": "auto"
},
"pluginVersion": "8.4.5",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "CHsKN4Unz"
},
"exemplar": true,
"expr": "mysql_uptime{ident=\"$host\"}",
"interval": "",
"legendFormat": "",
"refId": "A"
}
],
"title": "运行时间",
"type": "stat"
},
{
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "red",
"value": null
},
{
"color": "green",
"value": 2
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 3,
"x": 4,
"y": 0
},
"id": 20,
"options": {
"colorMode": "value",
"graphMode": "none",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto"
},
"pluginVersion": "8.4.5",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "CHsKN4Unz"
},
"exemplar": true,
"expr": "mysql_variables_innodb_flush_log_at_trx_commit{ident=\"$host\"}",
"interval": "",
"legendFormat": "",
"refId": "A"
}
],
"title": "Variables_innodb_flush_log_at_trx_commit",
"type": "stat"
},
{
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "red",
"value": null
},
{
"color": "green",
"value": 8388608
}
]
},
"unit": "bytes"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 3,
"x": 7,
"y": 0
},
"id": 18,
"options": {
"colorMode": "value",
"graphMode": "none",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto"
},
"pluginVersion": "8.4.5",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "CHsKN4Unz"
},
"exemplar": true,
"expr": "mysql_variables_max_allowed_packet{ident=\"$host\"}",
"interval": "",
"legendFormat": "",
"refId": "A"
}
],
"title": "Variables_max_allowed_packet",
"type": "stat"
},
{
"description": "",
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "red",
"value": null
},
{
"color": "green",
"value": 2097152
}
]
},
"unit": "bytes"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 3,
"x": 10,
"y": 0
},
"id": 14,
"options": {
"colorMode": "value",
"graphMode": "none",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto"
},
"pluginVersion": "8.4.5",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "CHsKN4Unz"
},
"exemplar": true,
"expr": "mysql_variables_sort_buffer_size{ident=\"$host\"}",
"interval": "",
"legendFormat": "",
"refId": "A"
}
],
"title": "Variables_sort_buffer_size",
"type": "stat"
},
{
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "red",
"value": null
},
{
"color": "green",
"value": 2097152
}
]
},
"unit": "bytes"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 3,
"x": 13,
"y": 0
},
"id": 16,
"options": {
"colorMode": "value",
"graphMode": "none",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto"
},
"pluginVersion": "8.4.5",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "CHsKN4Unz"
},
"exemplar": true,
"expr": "mysql_variables_join_buffer_size{ident=\"$host\"}",
"interval": "",
"legendFormat": "",
"refId": "A"
}
],
"title": "Variables_join_buffer_size",
"type": "stat"
},
{
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "red",
"value": null
},
{
"color": "green",
"value": 536870912
}
]
},
"unit": "bytes"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 4,
"x": 16,
"y": 0
},
"id": 6,
"options": {
"colorMode": "value",
"graphMode": "none",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto"
},
"pluginVersion": "8.4.5",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "CHsKN4Unz"
},
"exemplar": true,
"expr": "mysql_variables_innodb_buffer_pool_size{ident=\"$host\"}",
"interval": "",
"legendFormat": "",
"refId": "A"
}
],
"title": "Variables_innodb_buffer_pool_size",
"type": "stat"
},
{
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "red",
"value": null
},
{
"color": "green",
"value": 0.95
}
]
},
"unit": "percentunit"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 4,
"x": 20,
"y": 0
},
"id": 22,
"options": {
"colorMode": "value",
"graphMode": "none",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto"
},
"pluginVersion": "8.4.5",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "CHsKN4Unz"
},
"exemplar": true,
"expr": "(mysql_innodb_buffer_pool_read_requests{ident=\"$host\"} - mysql_innodb_buffer_pool_reads{ident=\"$host\"}) / mysql_innodb_buffer_pool_read_requests{ident=\"$host\"}",
"interval": "",
"legendFormat": "",
"refId": "A"
}
],
"title": "缓存命中率",
"type": "stat"
},
{
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 37,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "light-blue",
"value": null
}
]
},
"unit": "bytes"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 8,
"x": 0,
"y": 4
},
"id": 32,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "CHsKN4Unz"
},
"exemplar": true,
"expr": "procstat_memory_data{ident=\"$host\",user=\"mysql\"}",
"interval": "",
"legendFormat": "memory_data",
"refId": "A"
}
],
"title": "内存占用",
"type": "timeseries"
},
{
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 32,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineStyle": {
"fill": "solid"
},
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "light-blue",
"value": null
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 8,
"x": 8,
"y": 4
},
"id": 34,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "CHsKN4Unz"
},
"exemplar": true,
"expr": "procstat_cpu_usage{ident=\"$host\",user=\"mysql\"}",
"interval": "",
"legendFormat": "cpu_usage",
"refId": "A"
}
],
"title": "cpu使用",
"type": "timeseries"
},
{
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 59,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "light-green",
"value": null
}
]
},
"unit": "none"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 8,
"x": 16,
"y": 4
},
"id": 8,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "8.4.5",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "CHsKN4Unz"
},
"exemplar": true,
"expr": "mysql_process_list_threads_idle{ident=\"$host\"}",
"interval": "",
"legendFormat": "process_list",
"refId": "A"
}
],
"title": "进程数",
"type": "timeseries"
},
{
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 34,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineStyle": {
"fill": "solid"
},
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "light-blue",
"value": null
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 8,
"x": 0,
"y": 12
},
"id": 26,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "CHsKN4Unz"
},
"exemplar": true,
"expr": "mysql_threads_created{ident=\"$host\"}",
"interval": "",
"legendFormat": "threads_created",
"refId": "A"
}
],
"title": "已创建线程",
"type": "timeseries"
},
{
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 40,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "light-blue",
"value": null
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 8,
"x": 8,
"y": 12
},
"id": 28,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "CHsKN4Unz"
},
"exemplar": true,
"expr": "mysql_threads_running{ident=\"$host\"}",
"interval": "",
"legendFormat": "threads_running",
"refId": "A"
}
],
"title": "在运行线程",
"type": "timeseries"
},
{
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 42,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "light-blue",
"value": null
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 8,
"x": 16,
"y": 12
},
"id": 30,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "CHsKN4Unz"
},
"exemplar": true,
"expr": "mysql_threads_connected{ident=\"$host\"}",
"interval": "",
"legendFormat": "threads_connected",
"refId": "A"
}
],
"title": "已连接线程",
"type": "timeseries"
},
{
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds",
"seriesBy": "last"
},
"custom": {
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 1,
"drawStyle": "line",
"fillOpacity": 52,
"gradientMode": "hue",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": true,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "light-blue",
"value": null
}
]
},
"unit": "bytes"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 8,
"x": 0,
"y": 20
},
"id": 10,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "8.4.5",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "CHsKN4Unz"
},
"exemplar": true,
"expr": "mysql_innodb_buffer_pool_bytes_data{ident=\"$host\"}",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "innodb_buffer_pool_bytes_data",
"refId": "A"
}
],
"title": "缓存池数据大小",
"type": "timeseries"
},
{
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 50,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "light-green",
"value": null
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 8,
"x": 8,
"y": 20
},
"id": 24,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "CHsKN4Unz"
},
"exemplar": true,
"expr": "mysql_slow_queries{ident=\"$host\"}",
"interval": "",
"legendFormat": "slow_queries",
"refId": "A"
}
],
"title": "慢查询",
"type": "timeseries"
}
]
这篇关于Nginx日志统计展示的文章就介绍到这儿，希望我们推荐的文章对大家有所帮助，也希望大家多多支持为之网！
- 2024-10-29Nginx发布学习：从入门到实践的简单教程
- 2024-10-28Nginx发布：新手入门教程
- 2024-10-21nginx 怎么设置文件上传最大20M限制-icode9专业技术文章分享
- 2024-10-17关闭 nginx的命令是什么?-icode9专业技术文章分享
- 2024-09-17Nginx实用篇：实现负载均衡、限流与动静分离
- 2024-08-21宝塔nginx新增8022端口方法步骤-icode9专业技术文章分享
- 2024-08-21nginx配置，让ws升级为wss访问的方法步骤-icode9专业技术文章分享
- 2024-08-15nginx ws代理配置方法步骤-icode9专业技术文章分享
- 2024-08-14nginx 让访问带有/relid的地址返回404 ,例子 /relid-x-0.36-y-131.html-icode9专业技术文章分享
- 2024-08-14nginx 判断地址有/statics/的路径，指向到/home/html/statics/目录-icode9专业技术文章分享
