首页 > 代码库 > 日志分析 第六章 安装elasticsearch

日志分析 第六章 安装elasticsearch

在这里,以两台es集群为例。

es集群健康状况有三种状态,这里我们搭建的es集群,只要两台不同时挂掉,数据不会丢失。

green 所有主要分片和复制分片都可用
yellow 所有主要分片可用,但不是所有复制分片都可用
red 不是所有的主要分片都可用

 

 

 

 

举个例子:

  • 比如说现在集群节点es1位主节点,es2位复制分片节点,默认情况下,两台es都接收logstash传过来的日志,是负载均衡的。
  • 如果es1宕掉,es2会被提升为主节点,只有es2接收logstash传来得日志数据,同时整个集群状态由green转为red。
  • 修复es1,es1会作为分片节点加入集群中,es1加入到集群中后,会进行修复,es2中新增的数据同步到es1分片节点,同步过程整个集群健康状态转为yellow。
  • 当同步完成,集群状态转为green。

安装elasticsearch

官网:https://www.elastic.co/products/elasticsearch

# tar xf elasticsearch-2.3.5.tar.gz -C /usr/local/app/
# ln -sv /usr/local/app/elasticsearch-2.3.5 /usr/local/elasticsearch
# chown –R nobody.nobody /usr/local/app/elasticsearch-2.3.5/
# cd /usr/local/elasticsearch

编辑配置文件

es1

技术分享
cluster.name: myES
node.name: es1
node.master: true
path.data: /data/elasticsearch/data
path.logs: /data/elasticsearch/log
bootstrap.mlockall: True
network.host: 10.80.2.83
http.port: 9200
http.enabled: true
transport.tcp.port: 9300
transport.tcp.compress: true
discovery.zen.ping.unicast.hosts: ["10.80.2.83:9300","10.80.2.84:9300"]
discovery.zen.minimum_master_nodes: 1
discovery.zen.ping.timeout: 3s
discovery.zen.ping.multicast.enabled: false
gateway.recover_after_nodes: 1
gateway.recover_after_time: 5m
gateway.expected_nodes: 2
cluster.routing.allocation.node_initial_primaries_recoveries: 4
indices.recovery.max_size_per_sec: 50mb

index :
  analysis :
    analyzer :
      default :
        tokenizer : keyword
config/elasticsearch.yml
技术分享
#!/bin/sh

# check in case a user was using this mechanism
if [ "x$ES_CLASSPATH" != "x" ]; then
    cat >&2 << EOF
Error: Dont modify the classpath with ES_CLASSPATH. Best is to add
additional elements via the plugin mechanism, or if code must really be
added to the main classpath, add jars to lib/ (unsupported).
EOF
    exit 1
fi

ES_CLASSPATH="$ES_HOME/lib/elasticsearch-2.3.4.jar:$ES_HOME/lib/*"

if [ "x$ES_MIN_MEM" = "x" ]; then
    ES_MIN_MEM=8g
fi
if [ "x$ES_MAX_MEM" = "x" ]; then
    ES_MAX_MEM=8g
fi
if [ "x$ES_HEAP_SIZE" != "x" ]; then
    ES_MIN_MEM=$ES_HEAP_SIZE
    ES_MAX_MEM=$ES_HEAP_SIZE
fi

# min and max heap sizes should be set to the same value to avoid
# stop-the-world GC pauses during resize, and so that we can lock the
# heap in memory on startup to prevent any of it from being swapped
# out.
JAVA_OPTS="$JAVA_OPTS -Xms${ES_MIN_MEM}"
JAVA_OPTS="$JAVA_OPTS -Xmx${ES_MAX_MEM}"

# new generation
if [ "x$ES_HEAP_NEWSIZE" != "x" ]; then
    JAVA_OPTS="$JAVA_OPTS -Xmn${ES_HEAP_NEWSIZE}"
fi

# max direct memory
if [ "x$ES_DIRECT_SIZE" != "x" ]; then
    JAVA_OPTS="$JAVA_OPTS -XX:MaxDirectMemorySize=${ES_DIRECT_SIZE}"
fi

# set to headless, just in case
JAVA_OPTS="$JAVA_OPTS -Djava.awt.headless=true"

# Force the JVM to use IPv4 stack
if [ "x$ES_USE_IPV4" != "x" ]; then
  JAVA_OPTS="$JAVA_OPTS -Djava.net.preferIPv4Stack=true"
fi

# Add gc options. ES_GC_OPTS is unsupported, for internal testing
if [ "x$ES_GC_OPTS" = "x" ]; then
  ES_GC_OPTS="$ES_GC_OPTS -XX:+UseParNewGC"
  ES_GC_OPTS="$ES_GC_OPTS -XX:+UseConcMarkSweepGC"
  ES_GC_OPTS="$ES_GC_OPTS -XX:CMSInitiatingOccupancyFraction=75"
  ES_GC_OPTS="$ES_GC_OPTS -XX:+UseCMSInitiatingOccupancyOnly"
fi

JAVA_OPTS="$JAVA_OPTS $ES_GC_OPTS"

# GC logging options
if [ -n "$ES_GC_LOG_FILE" ]; then
  JAVA_OPTS="$JAVA_OPTS -XX:+PrintGCDetails"
  JAVA_OPTS="$JAVA_OPTS -XX:+PrintGCTimeStamps"
  JAVA_OPTS="$JAVA_OPTS -XX:+PrintGCDateStamps"
  JAVA_OPTS="$JAVA_OPTS -XX:+PrintClassHistogram"
  JAVA_OPTS="$JAVA_OPTS -XX:+PrintTenuringDistribution"
  JAVA_OPTS="$JAVA_OPTS -XX:+PrintGCApplicationStoppedTime"
  JAVA_OPTS="$JAVA_OPTS -Xloggc:$ES_GC_LOG_FILE"

  # Ensure that the directory for the log file exists: the JVM will not create it.
  mkdir -p "`dirname \"$ES_GC_LOG_FILE\"`"
fi

# Causes the JVM to dump its heap on OutOfMemory.
JAVA_OPTS="$JAVA_OPTS -XX:+HeapDumpOnOutOfMemoryError"
# The path to the heap dump location, note directory must exists and have enough
# space for a full heap dump.
#JAVA_OPTS="$JAVA_OPTS -XX:HeapDumpPath=$ES_HOME/logs/heapdump.hprof"

# Disables explicit GC
JAVA_OPTS="$JAVA_OPTS -XX:+DisableExplicitGC"

# Ensure UTF-8 encoding by default (e.g. filenames)
JAVA_OPTS="$JAVA_OPTS -Dfile.encoding=UTF-8"

# Use our provided JNA always versus the system one
JAVA_OPTS="$JAVA_OPTS -Djna.nosys=true"
bin/elasticsearch.in.sh

es2

技术分享
#集群名字
cluster.name: myES
#节点名字
node.name: es2
#是否有资格成为主节点
node.master: true
#数据目录
path.data: /data/elasticsearch/data
#日志目录
path.logs: /data/elasticsearch/log
#锁住内存
bootstrap.mlockall: True
#绑定的IP地址,可为0.0.0.0
#network.bind_host: 10.80.2.84
#该节点与其他节点交互的ip地址,
#network.publish_host: 10.80.2.84
#上面两个参数集合
network.host: 10.80.2.84
#对外服务端口
http.port: 9200
#是否使用http协议对外提供服务
http.enabled: true
#节点间交互tcp端口
transport.tcp.port: 9300
#传输数据时压缩
transport.tcp.compress: true
#集群中主节点初始化列表,通过这些节点自动发现加入其他节点
discovery.zen.ping.unicast.hosts: ["10.80.2.83:9300","10.80.2.84:9300"]
#保证集群中节点可以知道其他N各有主节点资格,默认1,es数超过2个,可设置大一些
discovery.zen.minimum_master_nodes: 1
#自动发现其他节点时ping超时时间,网络环境差,提高此参数
discovery.zen.ping.timeout: 3s
#是否打开多播发现节点,默认true
discovery.zen.ping.multicast.enabled: false
#集群中几个节点启动时进行数据恢复
gateway.recover_after_nodes: 1
#初始化数据恢复进程超时时间
gateway.recover_after_time: 5m
#这个es集群中节点数量,一旦这些数目节点启动,进行数据恢复
gateway.expected_nodes: 2
#初始化数据恢复时,并发恢复的线程个数
cluster.routing.allocation.node_initial_primaries_recoveries: 4
#数据恢复时最大带宽
indices.recovery.max_size_per_sec: 50mb
# 禁止分词,
index :
  analysis :
    analyzer :
      default :
        tokenizer : keyword
config/elasticsearch.yml
技术分享
#!/bin/sh

# check in case a user was using this mechanism
if [ "x$ES_CLASSPATH" != "x" ]; then
    cat >&2 << EOF
Error: Dont modify the classpath with ES_CLASSPATH. Best is to add
additional elements via the plugin mechanism, or if code must really be
added to the main classpath, add jars to lib/ (unsupported).
EOF
    exit 1
fi

ES_CLASSPATH="$ES_HOME/lib/elasticsearch-2.3.4.jar:$ES_HOME/lib/*"

if [ "x$ES_MIN_MEM" = "x" ]; then
    ES_MIN_MEM=8g
fi
if [ "x$ES_MAX_MEM" = "x" ]; then
    ES_MAX_MEM=8g
fi
if [ "x$ES_HEAP_SIZE" != "x" ]; then
    ES_MIN_MEM=$ES_HEAP_SIZE
    ES_MAX_MEM=$ES_HEAP_SIZE
fi

# min and max heap sizes should be set to the same value to avoid
# stop-the-world GC pauses during resize, and so that we can lock the
# heap in memory on startup to prevent any of it from being swapped
# out.
JAVA_OPTS="$JAVA_OPTS -Xms${ES_MIN_MEM}"
JAVA_OPTS="$JAVA_OPTS -Xmx${ES_MAX_MEM}"

# new generation
if [ "x$ES_HEAP_NEWSIZE" != "x" ]; then
    JAVA_OPTS="$JAVA_OPTS -Xmn${ES_HEAP_NEWSIZE}"
fi

# max direct memory
if [ "x$ES_DIRECT_SIZE" != "x" ]; then
    JAVA_OPTS="$JAVA_OPTS -XX:MaxDirectMemorySize=${ES_DIRECT_SIZE}"
fi

# set to headless, just in case
JAVA_OPTS="$JAVA_OPTS -Djava.awt.headless=true"

# Force the JVM to use IPv4 stack
if [ "x$ES_USE_IPV4" != "x" ]; then
  JAVA_OPTS="$JAVA_OPTS -Djava.net.preferIPv4Stack=true"
fi

# Add gc options. ES_GC_OPTS is unsupported, for internal testing
if [ "x$ES_GC_OPTS" = "x" ]; then
  ES_GC_OPTS="$ES_GC_OPTS -XX:+UseParNewGC"
  ES_GC_OPTS="$ES_GC_OPTS -XX:+UseConcMarkSweepGC"
  ES_GC_OPTS="$ES_GC_OPTS -XX:CMSInitiatingOccupancyFraction=75"
  ES_GC_OPTS="$ES_GC_OPTS -XX:+UseCMSInitiatingOccupancyOnly"
fi

JAVA_OPTS="$JAVA_OPTS $ES_GC_OPTS"

# GC logging options
if [ -n "$ES_GC_LOG_FILE" ]; then
  JAVA_OPTS="$JAVA_OPTS -XX:+PrintGCDetails"
  JAVA_OPTS="$JAVA_OPTS -XX:+PrintGCTimeStamps"
  JAVA_OPTS="$JAVA_OPTS -XX:+PrintGCDateStamps"
  JAVA_OPTS="$JAVA_OPTS -XX:+PrintClassHistogram"
  JAVA_OPTS="$JAVA_OPTS -XX:+PrintTenuringDistribution"
  JAVA_OPTS="$JAVA_OPTS -XX:+PrintGCApplicationStoppedTime"
  JAVA_OPTS="$JAVA_OPTS -Xloggc:$ES_GC_LOG_FILE"

  # Ensure that the directory for the log file exists: the JVM will not create it.
  mkdir -p "`dirname \"$ES_GC_LOG_FILE\"`"
fi

# Causes the JVM to dump its heap on OutOfMemory.
JAVA_OPTS="$JAVA_OPTS -XX:+HeapDumpOnOutOfMemoryError"
# The path to the heap dump location, note directory must exists and have enough
# space for a full heap dump.
#JAVA_OPTS="$JAVA_OPTS -XX:HeapDumpPath=$ES_HOME/logs/heapdump.hprof"

# Disables explicit GC
JAVA_OPTS="$JAVA_OPTS -XX:+DisableExplicitGC"

# Ensure UTF-8 encoding by default (e.g. filenames)
JAVA_OPTS="$JAVA_OPTS -Dfile.encoding=UTF-8"

# Use our provided JNA always versus the system one
JAVA_OPTS="$JAVA_OPTS -Djna.nosys=true"
bin/elasticsearch.in.sh

 

日志分析 第六章 安装elasticsearch