首页 > 代码库 > 日志分析 第五章 安装logstash
日志分析 第五章 安装logstash
logstash是java应用,依赖JDK,首先需要安装JDK,在安装jdk过程中,logstash-2.3.4使用JDK-1.7版本有bug,使用JDK-1.8版本正常,因此我们安装JDK-1.8版本。
安装JDK
官网地址:http://www.oracle.com/technetwork/java/javase/downloads/jdk8-downloads-2133151.html
# rpm -ivh jdk-8u101-linux-x64.rpm # echo "export JAVA_HOME=/usr/java/latest" >> /etc/profile # echo "export PATH=$PATH:$JAVA_HOME/bin" >> /etc/profile # source /etc/profile # java -version java version "1.8.0_101" Java(TM) SE Runtime Environment (build 1.8.0_101-b13) Java HotSpot(TM) 64-Bit Server VM (build 25.101-b13, mixed mode)
安装logstash
官网地址:https://www.elastic.co/products/logstash
# tar xf logstash-2.3.4.tar.gz -C /usr/local/app/ # ln -sv /usr/local/app/logstash-2.3.4 /usr/local/logstash # cd /usr/local/logstash
# mkdir patterns
NGUSERNAME [a-zA-Z\.\@\-\+_%]+ NGUSER %{NGUSERNAME} NGINXACCESS %{IPORHOST:server_name} %{IPORHOST:server_ip} %{IPORHOST:client_ip} %{NUMBER:client_port} \[%{HTTPDATE:timestamp}\] "%{WORD:verb} %{URIPATHPARAM:request} HTTP/%{NUMBER:httpversion}" %{NUMBER:response} (?:%{NUMBER:body_bytes_sent}|-) (?:"(?:%{URI:referrer}|-)"|%{QS:referrer}) %{QS:agent} %{NUMBER:request_time} (?:%{NUMBER:upstream_response_time}|-)
SYSLOGBASE2 (?:%{SYSLOGTIMESTAMP:timestamp}|%{TIMESTAMP_ISO8601:timestamp8601}) (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource} %{SYSLOGPROG}: SYSLOGPAMSESSION %{SYSLOGBASE} (?=%{GREEDYDATA:message})%{WORD:pam_module}\(%{DATA:pam_caller}\): session %{WORD:pam_session_state} for user %{USERNAME:username}(?: by %{GREEDYDATA:pam_by})? CRON_ACTION [A-Z ]+ CRONLOG %{SYSLOGBASE} \(%{USER:user}\) %{CRON_ACTION:action} \(%{DATA:message}\) SYSLOGLINE %{SYSLOGBASE2} %{GREEDYDATA:message} # IETF 5424 syslog(8) format (see http://www.rfc-editor.org/info/rfc5424) SYSLOG5424PRI <%{NONNEGINT:syslog5424_pri}> SYSLOG5424SD \[%{DATA}\]+ SYSLOG5424BASE %{SYSLOG5424PRI}%{NONNEGINT:syslog5424_ver} +(?:%{TIMESTAMP_ISO8601:syslog5424_ts}|-) +(?:%{HOSTNAME:syslog5424_host}|-) +(?:%{WORD:syslog5424_app}|-) +(?:%{WORD:syslog5424_proc}|-) +(?:%{WORD:syslog5424_msgid}|-) +(?:%{SYSLOG5424SD:syslog5424_sd}|-|) SYSLOG5424LINE %{SYSLOG5424BASE} +%{GREEDYDATA:syslog5424_msg}
编写配置文件
配置文件编写是一个难点,这里有一些示例供参考:https://github.com/logstash-plugins/logstash-patterns-core/tree/master/patterns
input { beats { port => 5048 host => "10.80.2.181" } } filter { if [type] == "51-nginxaccesslog" { grok { patterns_dir => ["./patterns"] match => { "message" => "%{IPORHOST:server_name} %{IPORHOST:server_ip} %{IPORHOST:clientip} %{NUMBER:clientport} \[%{HTTPDATE:timestamp}\] \"(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})\" %{NUMBER:status} (?:%{NUMBER:bytes}|-) %{QS:referrer} %{QS:agent} %{NUMBER:request_time:float} (?:%{NUMBER:upstream_time:float}|-)" } remove_field => ["message"] } date { match => [ "timestamp", "dd/MMM/YYYY:HH:mm:ss Z"] } } else if [type] == "51-nginxerrorlog" { grok { patterns_dir => ["./patterns"] match => { "message" => "%{DATESTAMP} %{SYSLOG5424SD:nginx_error_level} %{GREEDYDATA:nginx_error_msg}"} remove_field => ["message"] } date { match => [ "timestamp", "YYYY/MMM/dd HH:mm:ss"] } } else if [type] == "51-phperrorlog" { grok { patterns_dir => ["./patterns"] match => { "message" => "%{SYSLOG5424SD} (?:%{DATA:php_error_level}\:) %{GREEDYDATA:error_msg}" } remove_field => ["message"] } date { match => [ "timestamp", "dd-MMM-YYYY HH:mm:ss Z"] } } } output { if "_grokparsefailure" in [tags] { file { path => "/var/log/logstash/grokparsefailure-%{[type]}-%{+YYYY.MM.dd}.log" } } elasticsearch { hosts => ["10.80.2.83:9200","10.80.2.84:9200"] sniffing => true manage_template => false template_overwrite => true index => "%{[type]}-%{+YYYY.MM.dd}" document_type => "%{[type]}" } }
编写启动脚本
#!/bin/sh # Init script for logstash # Maintained by Elasticsearch # Generated by pleaserun. # Implemented based on LSB Core 3.1: # * Sections: 20.2, 20.3 # ### BEGIN INIT INFO # Provides: logstash # Required-Start: $remote_fs $syslog # Required-Stop: $remote_fs $syslog # Default-Start: 2 3 4 5 # Default-Stop: 0 1 6 # Short-Description: # Description: Starts Logstash as a daemon. ### END INIT INFO PATH=/sbin:/usr/sbin:/bin:/usr/bin export PATH if [ `id -u` -ne 0 ]; then echo "You need root privileges to run this script" exit 1 fi name=logstash pidfile="/usr/local/logstash/$name.pid" LS_USER=nobody LS_GROUP=nobody LS_HOME=/usr/local/logstash #LS_HOME=/home/logstash LS_HEAP_SIZE="12g" LS_LOG_DIR=/data/logstash/log LS_LOG_FILE="${LS_LOG_DIR}/$name.log" LS_CONF_DIR=/usr/local/logstash/conf.d LS_OPEN_FILES=65535 LS_NICE=-20 LS_THREADS=8 KILL_ON_STOP_TIMEOUT=${KILL_ON_STOP_TIMEOUT-0} #default value is zero to this variable but could be updated by user request LS_OPTS="" [ -r /etc/default/$name ] && . /etc/default/$name [ -r /etc/sysconfig/$name ] && . /etc/sysconfig/$name program=/usr/local/logstash/bin/logstash args="agent -f ${LS_CONF_DIR} -w ${LS_THREADS} -l ${LS_LOG_FILE} ${LS_OPTS}" quiet() { "$@" > /dev/null 2>&1 return $? } start() { LS_JAVA_OPTS="${LS_JAVA_OPTS} -Djava.io.tmpdir=${LS_HOME}" HOME=${LS_HOME} export PATH HOME LS_HEAP_SIZE LS_JAVA_OPTS LS_USE_GC_LOGGING LS_GC_LOG_FILE # chown doesn‘t grab the suplimental groups when setting the user:group - so we have to do it for it. # Boy, I hope we‘re root here. SGROUPS=$(id -Gn "$LS_USER" | tr " " "," | sed ‘s/,$//‘; echo ‘‘) if [ ! -z $SGROUPS ] then EXTRA_GROUPS="--groups $SGROUPS" fi # set ulimit as (root, presumably) first, before we drop privileges ulimit -n ${LS_OPEN_FILES} # Run the program! nice -n ${LS_NICE} chroot --userspec $LS_USER:$LS_GROUP $EXTRA_GROUPS / sh -c " cd $LS_HOME ulimit -n ${LS_OPEN_FILES} exec \"$program\" $args " > "${LS_LOG_DIR}/$name.stdout" 2> "${LS_LOG_DIR}/$name.err" & # Generate the pidfile from here. If we instead made the forked process # generate it there will be a race condition between the pidfile writing # and a process possibly asking for status. echo $! > $pidfile echo "$name started." return 0 } stop() { # Try a few times to kill TERM the program if status ; then pid=`cat "$pidfile"` echo "Killing $name (pid $pid) with SIGTERM" kill -TERM $pid # Wait for it to exit. for i in 1 2 3 4 5 6 7 8 9 ; do echo "Waiting $name (pid $pid) to die..." status || break sleep 1 done if status ; then if [ $KILL_ON_STOP_TIMEOUT -eq 1 ] ; then echo "Timeout reached. Killing $name (pid $pid) with SIGKILL. This may result in data loss." kill -KILL $pid echo "$name killed with SIGKILL." else echo "$name stop failed; still running." return 1 # stop timed out and not forced fi else echo "$name stopped." fi fi } status() { if [ -f "$pidfile" ] ; then pid=`cat "$pidfile"` if kill -0 $pid > /dev/null 2> /dev/null ; then # process by this pid is running. # It may not be our pid, but that‘s what you get with just pidfiles. # TODO(sissel): Check if this process seems to be the same as the one we # expect. It‘d be nice to use flock here, but flock uses fork, not exec, # so it makes it quite awkward to use in this case. return 0 else return 2 # program is dead but pid file exists fi else return 3 # program is not running fi } reload() { if status ; then kill -HUP `cat "$pidfile"` fi } force_stop() { if status ; then stop status && kill -KILL `cat "$pidfile"` fi } configtest() { # Check if a config file exists if [ ! "$(ls -A ${LS_CONF_DIR}/* 2> /dev/null)" ]; then echo "There aren‘t any configuration files in ${LS_CONF_DIR}" return 1 fi HOME=${LS_HOME} export PATH HOME test_args="--configtest -f ${LS_CONF_DIR} ${LS_OPTS}" $program ${test_args} [ $? -eq 0 ] && return 0 # Program not configured return 6 } case "$1" in start) status code=$? if [ $code -eq 0 ]; then echo "$name is already running" else start code=$? fi exit $code ;; stop) stop ;; force-stop) force_stop ;; status) status code=$? if [ $code -eq 0 ] ; then echo "$name is running" else echo "$name is not running" fi exit $code ;; reload) reload ;; restart) quiet configtest RET=$? if [ ${RET} -ne 0 ]; then echo "Configuration error. Not restarting. Re-run with configtest parameter for details" exit ${RET} fi stop && start ;; configtest) configtest exit $? ;; *) echo "Usage: $SCRIPTNAME {start|stop|force-stop|status|reload|restart|configtest}" >&2 exit 3 ;; esac exit $?
# chomd +x /etc/init.d/logstash # chown –R nobody.nobody /usr/local/logstash # chkconfig --add logstash
日志分析 第五章 安装logstash
声明:以上内容来自用户投稿及互联网公开渠道收集整理发布,本网站不拥有所有权,未作人工编辑处理,也不承担相关法律责任,若内容有误或涉及侵权可进行投诉: 投诉/举报 工作人员会在5个工作日内联系你,一经查实,本站将立刻删除涉嫌侵权内容。