首页 > 代码库 > filebeat专题
filebeat专题
一、filebeat概述
Filebeat是一个日志文件托运工具，在你的服务器上安装客户端后，filebeat会监控日志目录或者指定的日志文件，追踪读取这些文件（追踪文件的变化，不停的读），并且转发这些信息到elasticsearch或者logstarsh中存放
工作流程
当你开启filebeat程序的时候，它会启动一个或多个探测器（prospectors）去检测你指定的日志目录或文件，对于探测器找出的每一个日志文件，filebeat启动收割进程（harvester），每一个收割进程读取一个日志文件的新内容，并发送这些新的日志数据到处理程序（spooler），处理程序会集合这些事件，最后filebeat会发送集合的数据到你指定的地点
二、filebeat的安装
[root@blog ~]# wget https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-5.3.0-x86_64.rpm
[root@blog ~]# rpm -ivh filebeat-5.3.0-x86_64.rpm
warning: filebeat-5.3.0-x86_64.rpm: Header V4 RSA/SHA512 Signature, key ID d88e42b4: NOKEY
Preparing... ########################################### [100%]
1:filebeat ########################################### [100%]
[root@blog ~]# cd /etc/filebeat/
[root@blog filebeat]# ll
total 80
-rw-r--r-- 1 root root 37050 Mar 23 11:23 filebeat.full.yml
-rw-r--r-- 1 root root 15641 Mar 23 11:23 filebeat.template-es2x.json
-rw-r--r-- 1 root root 12745 Mar 23 11:23 filebeat.template.json
-rw------- 1 root root 4196 Mar 23 11:23 filebeat.yml
[root@blog filebeat]# mv filebeat.yml filebeat.yml.bak----------备份配置
三、配置filebeat
案例：收集服务器上nginx日志和系统日志
[root@blog filebeat]# vim filebeat.yml
filebeat.prospectors:
- input_type: log
paths:
- /var/log/nginx/access.log
tags: ["nginx-accesslog"]
document_type: nginxaccess
- input_type: log
paths:
- /var/log/messages
tags: ["sys-messages"]
document_type: sysmessages
tags: ["nginx-test-194"]
output.logstash:
hosts: ["10.2.11.222:5044"]
解释：
input
1、path选项：filebeat发送给logstash的路径，多个日志可以使用*.log 通配
2、tags：会向log中添加一个标签，该标签可以提供给logstash用于区分不同客户端不同业务的log
3、document_type：标记，跟tags差不多，区别不同的日志来源
output：指定发送log到哪台服务器上的那个服务，默认输出到elasticsearch
[root@blog filebeat]# service filebeat start
Starting filebeat: 2017/05/08 09:12:35.118324 beat.go:285: INFO Home path: [/usr/share/filebeat] Config path: [/etc/filebeat] Data path: [/var/lib/filebeat] Logs path: [/var/log/filebeat]
2017/05/08 09:12:35.118355 beat.go:186: INFO Setup Beat: filebeat; Version: 5.3.0
2017/05/08 09:12:35.118506 metrics.go:23: INFO Metrics logging every 30s
2017/05/08 09:12:35.118589 logstash.go:90: INFO Max Retries set to: 3
2017/05/08 09:12:35.118690 outputs.go:108: INFO Activated logstash as output plugin.
2017/05/08 09:12:35.118863 publish.go:295: INFO Publisher name: blog
2017/05/08 09:12:35.119893 async.go:63: INFO Flush Interval set to: 1s
2017/05/08 09:12:35.119959 async.go:64: INFO Max Bulk Size set to: 2048
Config OK
[ OK ]
配置logstash收集日志（在10.2.11.222）----stdout测试输出
input {
beats {
port => 5044
}
}
output {
stdout {
codec => "rubydebug"
}
}
[root@app1 ~]# /opt/logstash/bin/logstash -f filebeat.conf
Settings: Default pipeline workers: 1
Pipeline main started
{
"message" => "08/May/2017:17:22:15 +0800|10.2.15.222|10.2.11.252|-|GET|/|HTTP/1.1|200|18|-|Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36|0.000|",
"@version" => "1",
"@timestamp" => "2017-05-08T09:22:25.193Z",
"input_type" => "log",
"beat" => {
"version" => "5.3.0",
"name" => "blog",
"hostname" => "blog"
},
"source" => "/var/log/nginx/access.log",
"type" => "nginxaccess",
"tags" => [
[0] "nginx-test-194",
[1] "nginx-accesslog",
[2] "beats_input_codec_plain_applied"
],
"offset" => 1094,
"host" => "blog"
}
[root@app1 ~]# cat file.conf
input {
beats {
port => 5044
}
}
filter {
if "nginx-accesslog" in [tags] {
grok {
match => { "message" => "%{HTTPDATE:timestamp}\|%{IP:remote_addr}\|%{IPORHOST:http_host}\|(?:%{DATA:http_x_forwarded_for}|-)\|%{DATA:request_method}\|%{DATA:request_uri}\|%{DATA:server_protocol}\|%{NUMBER:status}\|(?:%{NUMBER:body_bytes_sent}|-)\|(?:%{DATA:http_referer}|-)\|%{DATA:http_user_agent}\|(?:%{DATA:request_time}|-)\|"}
}
mutate {
convert => ["status","integer"]
convert => ["body_bytes_sent","integer"]
convert => ["request_time","float"]
}
geoip {
source=>"remote_addr"
}
date {
match => [ "timestamp","dd/MMM/YYYY:HH:mm:ss Z"]
}
useragent {
source=>"http_user_agent"
}
}
if "sys-messages" in [tags] {
grok {
match => { "message" => "%{SYSLOGLINE}" }
add_field => [ "received_at", "%{@timestamp}" ]
add_field => [ "received_from", "%{host}" ]
}
date {
match => [ "timestamp", "MMM d HH:mm:ss" ]
}
#ruby {
# code => "event[‘@timestamp‘] = event[‘@timestamp‘].getlocal"
#}
}
}
output {
stdout {
codec => "rubydebug"
}
}
测试
[root@app1 ~]# /opt/logstash/bin/logstash -f file.conf
Settings: Default pipeline workers: 1
Pipeline main started
{
"message" => "08/May/2017:17:35:04 +0800|10.2.15.222|10.2.11.252|-|GET|/|HTTP/1.1|200|18|-|Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36|0.000|",
"@version" => "1",
"@timestamp" => "2017-05-08T09:35:04.000Z",
"offset" => 1401,
"input_type" => "log",
"beat" => {
"name" => "blog",
"hostname" => "blog",
"version" => "5.3.0"
},
"type" => "nginxaccess",
"tags" => [
[0] "nginx-test-194",
[1] "nginx-accesslog",
[2] "beats_input_codec_plain_applied"
],
"source" => "/var/log/nginx/access.log",
"host" => "blog",
"timestamp" => "08/May/2017:17:35:04 +0800",
"remote_addr" => "10.2.15.222",
"http_host" => "10.2.11.252",
"http_x_forwarded_for" => "-",
"request_method" => "GET",
"request_uri" => "/",
"server_protocol" => "HTTP/1.1",
"status" => 200,
"body_bytes_sent" => 18,
"http_referer" => "-",
"http_user_agent" => "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36",
"request_time" => 0.0,
"name" => "Chrome",
"os" => "Windows 7",
"os_name" => "Windows 7",
"device" => "Other",
"major" => "57",
"minor" => "0",
"patch" => "2987"
}
测试输出到elasticsearch上
output {
elasticsearch {
hosts => ["10.2.11.249:9200"]
index => "logstash-%{type}-%{+YYYY.MM.dd}"
document_type => "%{type}"
}
}
[root@elk01 indices]# ll -----------查看elasticsearch服务上，出现该索引
total 56
drwxr-xr-x 8 elasticsearch elasticsearch 4096 May 8 17:42 logstash-nginxaccess-2017.05.08
kibana上查看
filebeat专题