#tar zxvf redis-2.8.20.tar.gz

#cd redis-2.8.20

#make

#make install

# wget https://download.elastic.co/elasticsearch/elasticsearch/elasticsearch-1.7.1.tar.gz

# mkdir /usr/local/elk

# tar zxf elasticsearch-1.7.1.tar.gz -C /usr/local/elk/

# ln -s /usr/local/elk/elasticsearch-1.7.1/bin/elasticsearch /usr/bin

cluster.name=es_cluster

node.name=node0

path.data=http://www.mamicode.com/data/elasticsearch/data

path.logs=/data/elasticsearch/logs

#当前hostname或IP，我这里是elk

network.host=elk

network.port=9200

cluster.name:elasticsearch

↑配置es的集群名称，默认是elasticsearch，es会自动发现在同一网段下的es，如果在同一网段下有多个集群，就可以用这个属性来区分不同的集群。

node.name:”FranzKafka”

↑节点名，默认随机指定一个name列表中名字，该列表在es的jar包中config文件夹里name.txt文件中，其中有很多作者添加的有趣名字。

node.master:true

↑指定该节点是否有资格被选举成为node，默认是true，es是默认集群中的第一台机器为master，如果这台机挂了就会重新选举master。

node.data:true

↑指定该节点是否存储索引数据，默认为true。

index.number_of_shards:5

↑设置默认索引分片个数，默认为5片。

index.number_of_replicas:1

↑设置默认索引副本个数，默认为1个副本。

path.conf:/path/to/conf

↑设置配置文件的存储路径，默认是es根目录下的config文件夹。

path.data:/path/to/data

↑设置索引数据的存储路径，默认是es根目录下的data文件夹，可以设置多个存储路径，用逗号隔开，例：

path.data:/path/to/data1,/path/to/data2

path.work:/path/to/work

↑设置临时文件的存储路径，默认是es根目录下的work文件夹。

path.logs:/path/to/logs

↑设置日志文件的存储路径，默认是es根目录下的logs文件夹

path.plugins:/path/to/plugins

↑设置插件的存放路径，默认是es根目录下的plugins文件夹

bootstrap.mlockall:true

↑设置为true来锁住内存。因为当jvm开始swapping时es的效率会降低，所以要保证它不swap，可以把ES_MIN_MEM和ES_MAX_MEM两个环境变量设置成同一个值，并且保证机器有足够的内存分配给es。同时也要允许elasticsearch的进程可以锁住内存，linux下可以通过`ulimit-lunlimited`命令。

network.bind_host:192.168.0.1

↑设置绑定的ip地址，可以是ipv4或ipv6的，默认为0.0.0.0。network.publish_host:192.168.0.1

设置其它节点和该节点交互的ip地址，如果不设置它会自动判断，值必须是个真实的ip地址。

network.host:192.168.0.1

↑这个参数是用来同时设置bind_host和publish_host上面两个参数。

transport.tcp.port:9300

↑设置节点间交互的tcp端口，默认是9300。

transport.tcp.compress:true

↑设置是否压缩tcp传输时的数据，默认为false，不压缩。

http.port:9200

↑设置对外服务的http端口，默认为9200。

http.max_content_length:100mb

↑设置内容的最大容量，默认100mb

http.enabled:false

↑是否使用http协议对外提供服务，默认为true，开启。

gateway.type:local

↑gateway的类型，默认为local即为本地文件系统，可以设置为本地文件系统，分布式文件系统，hadoop的HDFS，和amazon的s3服务器，其它文件系统的设置方法下次再详细说。

gateway.recover_after_nodes:1

↑设置集群中N个节点启动时进行数据恢复，默认为1。

gateway.recover_after_time:5m

↑设置初始化数据恢复进程的超时时间，默认是5分钟。

gateway.expected_nodes:2

↑设置这个集群中节点的数量，默认为2，一旦这N个节点启动，就会立即进行数据恢复。

cluster.routing.allocation.node_initial_primaries_recoveries:4

↑初始化数据恢复时，并发恢复线程的个数，默认为4。

cluster.routing.allocation.node_concurrent_recoveries:2

↑添加删除节点或负载均衡时并发恢复线程的个数，默认为4。

indices.recovery.max_size_per_sec:0

↑设置数据恢复时限制的带宽，如入100mb，默认为0，即无限制。

indices.recovery.concurrent_streams:5

↑设置这个参数来限制从其它分片恢复数据时最大同时打开并发流的个数，默认为5。

discovery.zen.minimum_master_nodes:1

↑设置这个参数来保证集群中的节点可以知道其它N个有master资格的节点。默认为1，对于大的集群来说，可以设置大一点的值（2-4）

discovery.zen.ping.timeout:3s

↑设置集群中自动发现其它节点时ping连接超时时间，默认为3秒，对于比较差的网络环境可以高点的值来防止自动发现时出错。

discovery.zen.ping.multicast.enabled:false

↑设置是否打开多播发现节点，默认是true。

discovery.zen.ping.unicast.hosts:[“host1″,”host2:port”,”host3[portX-portY]”]

↑设置集群中master节点的初始列表，可以通过这些节点来自动发现新加入集群的节点

# elasticsearch start -d

# curl -X GET http://localhost:9200

{

  "status" : 200,

  "name" : "Wasp",

  "cluster_name" : "elasticsearch",

  "version" : {

    "number" : "1.7.1",

    "build_hash" : "b88f43fc40b0bcd7f173a1f9ee2e97816de80b19",

    "build_timestamp" : "2015-07-29T09:54:16Z",

    "build_snapshot" : false,

    "lucene_version" : "4.10.4"

  },

  "tagline" : "You Know, for Search"

}

# wget https://download.elastic.co/logstash/logstash/logstash-1.5.3.tar.gz

# tar -zxf logstash-1.5.3.tar.gz -C /usr/local/elk

# mkdir /usr/local/elk/logstash-1.5.3/etc

# vi /etc/yum.repos.d/CentOS-Base.repo

 [logstash-1.5]

name=Logstash repository for 1.5.x packages

baseurl=http://packages.elasticsearch.org/logstash/1.5/centos

gpgcheck=1

gpgkey=http://packages.elasticsearch.org/GPG-KEY-elasticsearch

enabled=1

# yum install logstash    #yum安装会安装在/opt目录下

#./bin/logstash -e ‘input{stdin{}}output{stdout{codec=>rubydebug}}‘

Hello World

{

       "message" => "Hello World",

      "@version" => "1",

    "@timestamp" => "2017-03-21T06:45:59.349Z",

          "host" => "localhost.localdomain"

}

# cd <nginx_dir>/conf

# vi nginx.conf     #设置log_format,去掉注释

    log_format  main  ‘$remote_addr - $remote_user [$time_local] "$request" ‘

                      ‘$status $body_bytes_sent "$http_referer" ‘

                      ‘"$http_user_agent" "$http_x_forwarded_for"‘;

access_log  logs/host.access.log  main;  #设置access日志，有访问时自动写入此文件

# nginx -s reload

# vi httpd.conf

#设置如下日志格式

LogFormat "%h %l %u %t %D \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" \"%{Cookie}i\" \"%{Set-Cookie}o\"" combined

input {

        file {

                type => "apache"

                path => ["/log/access_log"]

          #↑多个源可以写成path => ["/var/log/messages","/var/log/*.log"]

        }

       file {

                type => "nginx"

                path => ["/usr/share/nginx/logs/access.log"]

        }

}

output {

        redis {

                host => "127.0.0.1" #redis server

                data_type => "list"

                key => "logstash:redis"

        }

}

input {

    file {

        #↓监听文件的路径

        path => ["/var/log/*","/log/test.log"]

        #排除不想监听的文件

        exclude => "1.log"

        #↓设置多长时间扫描目录，发现新文件

        discover_interval => 15

        #↓设置多长时间检测文件是否修改

        stat_interval => 1

        #↓监听文件的起始位置，默认是end

        start_position => beginning

        #↓监听文件读取信息记录的位置

        sincedb_path => "/log/add.log"

        #↓设置多长时间会写入读取的位置信息

        sincedb_write_interval => 15

    }

}

input {

#↓定义源为redis

        redis {

                host => "127.0.0.1"

                data_type => "list"

                key => "logstash:redis"

                type => "redis-input"

        }

}

#filter部分通过if语句根据不同的type进行匹配不同的grok规则.

filter {

    if [type] == "apache" {

    #↓定义apache日志的索引规则，type:apache已在logstash_agent.conf文件中定义.

        grok {

                match => [

    "message", "%{IPORHOST:http_host} (%{IPORHOST:client_ip}|-) - \[%{HTTPDATE:timestamp}\] \"(?:%{WORD:http_verb} %{NOTSPACE:http_request} (?: HTTP/%{NUMBER:http_version})?|%{DATA:raw_http_request})\" %{NUMBER:http_status_code} (?:%{NUMBER:bytes_read}|-) %{QS:referrer} %{QS:agent} %{QS:cookie_in} (%{QS:cookie_out}|-)"

                ]

        }

}

    if [type] == "nginx" {

    #↓定义nginx日志的索引规则，type:nginx已在logstash_agent.conf文件中定义.

        grok {

                match => [

    "message", "%{IPORHOST:http_host} - - \[%{HTTPDATE:timestamp}\] \"(?:%{WORD:http_verb} %{NOTSPACE:http_request} HTTP/%{NUMBER:http_version}\" %{NUMBER:http_status_code} (?:%{NUMBER:bytes_read}|-)) %{QS:referrer} %{QS:agent} %{QS:forwarded}"

                ]

        }

    }

}

#↓output部分通过if语句根据不同的type输出成不同的索引，该索引需要在kibana中进行创建，见后续部分.

output {

    if [type] == "apache" {

        elasticsearch {

                embedded => false

                protocol => "http"

                host => "localhost"

                port => "9200"

                index => "apache_access"

        }

    }

if [type] == "nginx" {

        elasticsearch {

                embedded => false

                protocol => "http"

                host => "localhost"

                port => "9200"

                index => "nginx_access"

        }

    }

}

# Kibana is served by a back end server. This controls which port to use.

port: 5601

# The host to bind the server to.

host: "0.0.0.0"

# The Elasticsearch instance to use for all your queries.

elasticsearch_url: "http://localhost:9200"

# preserve_elasticsearch_host true will send the hostname specified in `elasticsearch`. If you set it to false,

# then the host you use to connect to *this* Kibana instance will be sent.

elasticsearch_preserve_host: true

# Kibana uses an index in Elasticsearch to store saved searches, visualizations

# and dashboards. It will create a new index if it doesn‘t already exist.

kibana_index: ".kibana"

# If your Elasticsearch is protected with basic auth, this is the user credentials

# used by the Kibana server to perform maintence on the kibana_index at statup. Your Kibana

# users will still need to authenticate with Elasticsearch (which is proxied thorugh

# the Kibana server)

# kibana_elasticsearch_username: user

# kibana_elasticsearch_password: pass

# If your Elasticsearch requires client certificate and key

# kibana_elasticsearch_client_crt: /path/to/your/client.crt

# kibana_elasticsearch_client_key: /path/to/your/client.key

# If you need to provide a CA certificate for your Elasticsarech instance, put

# the path of the pem file here.

# ca: /path/to/your/CA.pem

# The default application to load.

default_app_id: "discover"

# Time in milliseconds to wait for elasticsearch to respond to pings, defaults to

# request_timeout setting

# ping_timeout: 1500

# Time in milliseconds to wait for responses from the back end or elasticsearch.

# This must be > 0

request_timeout: 300000

# Time in milliseconds for Elasticsearch to wait for responses from shards.

# Set to 0 to disable.

shard_timeout: 0

# Time in milliseconds to wait for Elasticsearch at Kibana startup before retrying

# startup_timeout: 5000

# Set to false to have a complete disregard for the validity of the SSL

# certificate.

verify_ssl: true

# SSL for outgoing requests from the Kibana Server (PEM formatted)

# ssl_key_file: /path/to/your/server.key

# ssl_cert_file: /path/to/your/server.crt

# Set the path to where you would like the process id file to be created.

# pid_file: /var/run/kibana.pid

# If you would like to send the log output to a file you can set the path below.

# This will also turn off the STDOUT log output.

# log_file: ./kibana.log

# Plugins that are included in the build, and no longer found in the plugins/ folder

bundled_plugin_ids:

 - plugins/dashboard/index

 - plugins/discover/index

 - plugins/doc/index

 - plugins/kibana/index

 - plugins/markdown_vis/index

 - plugins/metric_vis/index

 - plugins/settings/index

 - plugins/table_vis/index

 - plugins/vis_types/index

 - plugins/visualize/index

#启动redis

#nohup redis-server /etc/redis/redis.conf >/etc/redis/nohup.out &

#启动elasticsearch

#elasticsearch start -d

#启动agent

#cd /usr/local/elk/logstash-1.5.3

nohup ./bin/logstash -f ./etc/logstash_agent.conf > agent.out &

#启动indexer

#cd /usr/local/elk/logstash-1.5.3

nohup ./bin/logstash -f ./etc/logstash_indexer.conf >indexer.out&

#启动kibana

#cd /usr/local/elk/kibana

nohup ./bin/kibana >kibana.out &