搭建ELK日志分析平台
单机安装
[root@es-0001 ~]# vim /etc/hosts
192.168.1.41 es-0001
[root@es-0001 ~]# yum install -y java-1.8.0-openjdk elasticsearch
[root@es-0001 ~]# vim /etc/elasticsearch/elasticsearch.yml
55: network.host: 0.0.0.0
[root@es-0001 ~]# systemctl enable --now elasticsearch
[root@es-0001 ~]# curl http://192.168.1.41:9200/
{
"name" : "War Eagle",
"cluster_name" : "elasticsearch",
"version" : {
"number" : "2.3.4",
"build_hash" : "e455fd0c13dceca8dbbdbb1665d068ae55dabe3f",
"build_timestamp" : "2016-06-30T11:24:31Z",
"build_snapshot" : false,
"lucene_version" : "5.5.0"
},
"tagline" : "You Know, for Search"
}
集群安装
[root@es-0001 ~]# vim /etc/hosts
192.168.1.41 es-0001
192.168.1.42 es-0002
192.168.1.43 es-0003
192.168.1.44 es-0004
192.168.1.45 es-0005
[root@es-0001 ~]# yum install -y java-1.8.0-openjdk elasticsearch
[root@es-0001 ~]# vim /etc/elasticsearch/elasticsearch.yml
17: cluster.name: my-es
23: node.name: es-0001 # 本机主机名
55: network.host: 0.0.0.0
68: discovery.zen.ping.unicast.hosts: ["es-0001", "es-0002"]
[root@es-0001 ~]# systemctl enable --now elasticsearch
[root@es-0001 ~]# curl http://192.168.1.41:9200/_cluster/health?pretty
{
"cluster_name" : "my-es",
"status" : "green",
"timed_out" : false,
"number_of_nodes" : 5,
"number_of_data_nodes" : 5,
... ...
}
插件安装（head）
安装 apache，并部署 head 插件
[root@web ~]# yum install -y httpd
[root@web ~]# tar zxf head.tar.gz
[root@web ~]# mv elasticsearch-head /var/www/html/head
[root@web ~]# systemctl enable --now httpd
Created symlink from /etc/systemd/system/multi-user.target.wants/httpd.service to /usr/lib/systemd/system/httpd.service.
[root@es-0001 ~]# vim /etc/elasticsearch/elasticsearch.yml
# 配置文件最后追加
http.cors.enabled : true
http.cors.allow-origin : "*"
http.cors.allow-methods : OPTIONS, HEAD, GET, POST, PUT, DELETE
http.cors.allow-headers : X-Requested-With,X-Auth-Token,Content-Type,Content-Length
[root@es-0001 ~]# systemctl restart elasticsearch.service
创建索引
指定索引的名称，指定分片数量，指定副本数量
创建索引使用 PUT 方法，创建完成以后通过 head 插件验证
[root@es-0001 ~]# curl -XPUT -H "Content-Type: application/json" http://es-0001:9200/tedu -d \
'{
"settings":{
"index":{
"number_of_shards": 5,
"number_of_replicas": 1
}
}
}'
增加数据
[root@es-0001 ~]# curl -XPUT -H "Content-Type: application/json" http://es-0001:9200/tedu/teacher/1 -d \
'{
"职业": "诗人",
"名字": "李白",
"称号": "诗仙",
"年代": "唐"
}'
查询数据
[root@es-0001 ~]# curl -XGET http://es-0001:9200/tedu/teacher/1?pretty
修改数据
[root@es-0001 ~]# curl -XPOST -H "Content-Type: application/json" \
http://es-0001:9200/tedu/teacher/1/_update -d '{ "doc": {"年代":"公元701"}}'
删除数据
# 删除一条
[root@es-0001 ~]# curl -XDELETE http://es-0001:9200/tedu/teacher/1
# 删除索引
[root@es-0001 ~]# curl -XDELETE http://es-0001:9200/tedu
[root@kibana ~]# vim /etc/hosts
192.168.1.41 es-0001
192.168.1.42 es-0002
192.168.1.43 es-0003
192.168.1.44 es-0004
192.168.1.45 es-0005
192.168.1.46 kibana
[root@kibana ~]# yum install -y kibana
[root@kibana ~]# vim /etc/kibana/kibana.yml
02 server.port: 5601
07 server.host: "0.0.0.0"
28 elasticsearch.hosts: ["http://es-0002:9200", "http://es-0003:9200"]
37 kibana.index: ".kibana"
40 kibana.defaultAppId: "home"
113 i18n.locale: "zh-CN"
[root@kibana ~]# systemctl enable --now kibana
[root@logstash ~]# vim /etc/hosts
192.168.1.41 es-0001
192.168.1.42 es-0002
192.168.1.43 es-0003
192.168.1.44 es-0004
192.168.1.45 es-0005
192.168.1.47 logstash
[root@logstash ~]# yum install -y java-1.8.0-openjdk logstash
[root@logstash ~]# ln -s /etc/logstash /usr/share/logstash/config
[root@logstash ~]# vim /etc/logstash/conf.d/my.conf
input {
stdin {}
}
filter{ }
output{
stdout{}
}
[root@logstash ~]# /usr/share/logstash/bin/logstash
插件与调试格式
使用json格式字符串测试 {"a":"1", "b":"2", "c":"3"}
[root@logstash ~]# vim /etc/logstash/conf.d/my.conf
input {
stdin { codec => "json" }
}
filter{ }
output{
stdout{ codec => "rubydebug" }
}
[root@logstash ~]# /usr/share/logstash/bin/logstash
官方手册地址
https://www.elastic.co/guide/en/logstash/current/index.html
input file插件
[root@logstash ~]# vim /etc/logstash/conf.d/my.conf
input {
file {
path => ["/tmp/c.log"]
type => "test"
start_position => "beginning"
sincedb_path => "/var/lib/logstash/sincedb"
}
}
filter{ }
output{
stdout{ codec => "rubydebug" }
}
[root@logstash ~]# rm -rf /var/lib/logstash/plugins/inputs/file/.sincedb_*
[root@logstash ~]# /usr/share/logstash/bin/logstash
filter grok插件
正则表达式宏调用格式: %{宏名称:名字}
宏文件路径
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-patterns-core-4.1.2/patterns
[root@logstash ~]# echo '192.168.1.252 - - [29/Jul/2020:14:06:57 +0800] "GET /info.html HTTP/1.1" 200 119 "-" "curl/7.29.0"' >/tmp/c.log
[root@logstash ~]# vim /etc/logstash/conf.d/my.conf
input {
file {
path => ["/tmp/c.log"]
type => "test"
start_position => "beginning"
sincedb_path => "/dev/null"
}
}
filter{
grok {
match => { "message" => "%{HTTPD_COMBINEDLOG}" }
}
}
output{
stdout{ codec => "rubydebug" }
}
[root@logstash ~]# /usr/share/logstash/bin/logstash
output elasticsearch插件
[root@logstash ~]# vim /etc/logstash/conf.d/my.conf
input {
file {
path => ["/tmp/c.log"]
type => "test"
start_position => "beginning"
sincedb_path => "/dev/null"
}
}
filter{
grok {
match => { "message" => "%{HTTPD_COMBINEDLOG}" }
}
}
output{
stdout{ codec => "rubydebug" }
elasticsearch {
hosts => ["es-0004:9200", "es-0005:9200"]
index => "weblog-%{+YYYY.MM.dd}"
}
}
[root@logstash ~]# /usr/share/logstash/bin/logstash
[root@logstash ~]# vim /etc/logstash/conf.d/my.conf
input {
stdin { codec => "json" }
file{
path => ["/tmp/c.log"]
type => "test"
start_position => "beginning"
sincedb_path => "/var/lib/logstash/sincedb"
}
beats {
port => 5044
}
}
filter{
grok {
match => { "message" => "%{HTTPD_COMBINEDLOG}" }
}
}
output{
stdout{ codec => "rubydebug" }
elasticsearch {
hosts => ["es-0004:9200", "es-0005:9200"]
index => "weblog-%{+YYYY.MM.dd}"
}
}
[root@logstash ~]# /usr/share/logstash/bin/logstash
web服务安装filebeat
[root@web ~]# yum install -y filebeat
[root@web ~]# vim /etc/filebeat/filebeat.yml
24: enabled: true
28: - /var/log/httpd/access_log
45: fields:
46: my_type: apache
148, 150 注释掉
161: output.logstash:
163: hosts: ["192.168.1.47:5044"]
180, 181, 182 注释掉
[root@web ~]# grep -Pv "^\s*(#|$)" /etc/filebeat/filebeat.yml
[root@web ~]# systemctl enable --now filebeat
