Cloudera Hadoop 5.2 安装

首页 > 代码库 > Cloudera Hadoop 5.2 安装

2024-08-07 13:40:55 219人阅读

部署笔记

环境准备

1）JDK

2）本地cdh yum源省.....

3）本地hosts

10.20.120.11 sp-kvm01.hz.idc.com

10.20.120.12 sp-kvm02.hz.idc.com

10.20.120.13 sp-kvm03.hz.idc.com

10.20.120.14 sp-kvm04.hz.idc.com

10.20.120.21 hd01.hz.idc.com

10.20.120.22 hd02.hz.idc.com

10.20.120.31 hd-node01.hz.idc.com

Ganglia 监控部署

参考文档.....

http://10.20.120.22/ganglia

Hadoop 部署

Zookeeper 安装

yum install zookeeper zookeeper-server

mkdir /opt/zookeeper

/etc/zookeeper/conf/zoo.cfg

maxClientCnxns=50

# The number of milliseconds of each tick

tickTime=2000

# The number of ticks that the initial

# synchronization phase can take

initLimit=10

# The number of ticks that can pass between

# sending a request and getting an acknowledgement

syncLimit=5

# the directory where the snapshot is stored.

dataDir=/opt/zookeeper

# the port at which the clients will connect

clientPort=2181

server.1=sp-kvm01.hz.idc.com:2888:3888

server.2=sp-kvm02.hz.idc.com:2888:3888

server.3=sp-kvm03.hz.idc.com:2888:3888

每个节点下创建myid文件

# echo 1 > /opt/zookeeper/myid #第二个节点2 第三个节点3

# chown -R zookeeper.zookeeper /data/zookeeper

初始换zookeeper

# /etc/init.d/zookeeper-server init --myid=1 #第二个节点2 第三个节点3

启动

/etc/init.d/zookeeper-server start

查看状态

[ops@sp-kvm01 ~]# zookeeper-server status

JMX enabled by default

Using config: /etc/zookeeper/conf/zoo.cfg

Mode: follower

Journalnode

#yum -y install hadoop-hdfs-journalnode

#mkdir -p /data/1/dfs/jn && chown -R hdfs:hdfs /data/1/dfs/jn

NameNode

#yum -y install hadoop-hdfs-namenode hadoop-hdfs-zkfc hadoop-client

DataNode

#yum -y install hadoop-hdfs-datanode

#set namenode datanode dir

#mkdir -p /data/1/dfs/nn /data/{1,2}/dfs/dn

#chown -R hdfs:hdfs /data/1/dfs/nn /data/{1,2}/dfs/dn

#chmod 700 /data/1/dfs/nn /data/{1,2}/dfs/dn

#chmod go-rx /data/1/dfs/nn

配置文件参考

<name>fs.defaultFS</name>

<value>hdfs://hzcluster</value>

</property>

<name>fs.trash.interval</name>

</property>

<name>io.file.buffer.size</name>

</property>

<name>io.compression.codecs</name>

<value>org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.BZip2Codec,org.apache.hadoop.io.compress.DeflateCodec,org.apache.hadoop.io.compress.SnappyCodec,org.apache.hadoop.io.compress.Lz4Codec</value>

</property>

<name>ha.zookeeper.quorum</name>

</property>

<name>ha.zookeeper.session-timeout.ms</name>

<description>指定ZooKeeper超时间隔，单位毫秒</description>

</property>

<name>hadoop.security.authentication</name>

<value>simple</value>

</property>

<name>hadoop.rpc.protection</name>

<value>authentication</value>

</property>

<name>hadoop.security.auth_to_local</name>

<value>DEFAULT</value>

</property>

<name>hadoop.proxyuser.httpfs.hosts</name>

</property>

<name>hadoop.proxyuser.httpfs.groups</name>

</property>

#vim /etc/hadoop/conf/hdfs-site.xml

<name>dfs.replication</name>

</property>

<name>dfs.blocksize</name>

</property>

<name>dfs.permissions.superusergroup</name>

<value>hadoop</value>

</property>

<name>dfs.namenode.name.dir</name>

</property>

<name>dfs.datanode.data.dir</name>

</property>

<name>dfs.nameservices</name>

<value>hzcluster</value>

</property>

<name>dfs.ha.namenodes.hzcluster</name>

</property>

<name>dfs.namenode.rpc-address.hzcluster.nn1</name>

</property>

<name>dfs.namenode.rpc-address.hzcluster.nn2</name>

</property>

<name>dfs.namenode.http-address.hzcluster.nn1</name>

</property>

<name>dfs.namenode.http-address.hzcluster.nn2</name>

</property>

<name>dfs.namenode.shared.edits.dir</name>

<value>qjournal://hd01.hz.idc.com:8485;hd02.hz.idc.com:8485;hd-node01.hz.idc.com:8485/hzcluster</value>

</property>

<name>dfs.journalnode.edits.dir</name>

</property>

<name>dfs.client.failover.proxy.provider.hzcluster</name>

<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>

</property>

<name>dfs.ha.fencing.methods</name>

<value>sshfence</value>

</property>

<name>dfs.ha.fencing.ssh.private-key-files</name>

<value>/var/lib/hadoop-hdfs/.ssh/id_rsa</value>

</property>

<name>dfs.ha.automatic-failover.enabled</name>

</property>

<name>ha.zookeeper.quorum</name>

</property>

<name>topology.script.file.name</name>

<value>/etc/hadoop/conf/RackAware.py</value>

</property>

<name>dfs.datanode.du.reserved</name>

<description>保留10G存储空间</description>

</property>

<name>dfs.namenode.handler.count</name>

<description>namenode的线程数</description>

</property>

<name>dfs.datanode.handler.count</name>

<description>datanode的线程数</description>

</property>

<name>dfs.datanode.max.xcievers</name>

<description>datanode打开的句柄数目</description>

</property>

<name>dfs.socket.timeout</name>

</property>

<name>dfs.datanode.socket.write.timeout</name>

</property>

<name>dfs.balance.bandwidthPerSec</name>

<description>balance的最大带宽占用</description>

</property>

<name>dfs.hosts.exclude</name>

<value>/etc/hadoop/conf/excludes</value>

</property>

</configuration>

vim /etc/hadoop/conf/hadoop-metrics2.properties

*.sink.ganglia.class=org.apache.hadoop.metrics2.sink.ganglia.GangliaSink31

*.sink.ganglia.period=10

*.sink.ganglia.slope=jvm.metrics.gcCount=zero,jvm.metrics.memHeapUsedM=both

*.sink.ganglia.dmax=jvm.metrics.threadsBlocked=70,jvm.metrics.memHeapUsedM=40

namenode.sink.ganglia.servers=10.20.120.22:8649

resourcemanager.sink.ganglia.servers=10.20.120.22:8649

datanode.sink.ganglia.servers=10.20.120.22:8649

nodemanager.sink.ganglia.servers=10.20.120.22:8649

maptask.sink.ganglia.servers=10.20.120.22:8649

reducetask.sink.ganglia.servers=10.20.120.22:8649

1)配置NameNode之间的sshfence

2)NameNode启动时会连接Journal Node： /etc/init.d/hadoop-hdfs-journalnode start

3)格式化(主）上的NameNode #sudo -u hdfs hadoop namenode -format

4）启动主 NN #service hadoop-hdfs-namenode start

从同步 #sudo -u hdfs hadoop namenode -bootstrapStandby

从启动 #service hadoop-hdfs-namenode start

5）配置自动切换

在任意一个NameNode上运行即可,会创建一个znode用于自动故障转移。

#hdfs zkfc -formatZK

#启动 zkfs（2个NN） #service hadoop-hdfs-zkfc start

6）启动datanode

添加tmp目录

#sudo -u hdfs hadoop fs -mkdir /tmp

#sudo -u hdfs hadoop fs -chmod -R 1777 /tmp

#hadoop fs -ls /

##############################################################

NameNode HA自动切换测试

##############################################################

查看namenode状态

[root@master01 ~]# sudo -u hdfs hdfs haadmin -getServiceState nn1

active

[root@master01 ~]# sudo -u hdfs hdfs haadmin -getServiceState nn2

standby

测试高可用

# ps -ef | grep namenode

hdfs 11499 1 1 17:27 ? 00:00:11 /usr/java/jdk1.6.0_31/bin/java

# kill -9 11499

查看状态

# sudo -u hdfs hdfs haadmin -getServiceState nn2

active

# /etc/init.d/hadoop-hdfs-namenode start

Starting Hadoop namenode: [ OK ]

# sudo -u hdfs hdfs haadmin -getServiceState nn1

standby

手动切换

# sudo -u hdfs hdfs haadmin -failover nn2 nn1

# sudo -u hdfs hdfs haadmin -getServiceState nn1

active

HBase 部署

#yum -y install hbase-master hbase-rest hbase-thrift

Reg

#yum -y install hbase-regionserver

创建hbase目录

# sudo -u hdfs hadoop fs -mkdir /hbase

# sudo -u hdfs hadoop fs -chown hbase /hbase

vim hbase-site.xml

<name>hbase.rest.port</name>

</property>

<name>hbase.cluster.distributed</name>

</property>

<name>hbase.rootdir</name>

<value>hdfs://hzcluster/hbase</value>

</property>

<name>hbase.zookeeper.quorum</name>

</property>

<name>hbase.zookeeper.property.clientPort</name>

</property>

<name>hbase.client.write.buffer</name>

</property>

<name>hbase.client.pause</name>

</property>

<name>hbase.client.retries.number</name>

</property>

<name>hbase.client.scanner.caching</name>

</property>

<name>hbase.client.keyvalue.maxsize</name>

</property>

<name>hbase.rpc.timeout</name>

</property>

<name>hbase.security.authentication</name>

<value>simple</value>

</property>

<name>zookeeper.session.timeout</name>

</property>

[root@hd01 conf]# cat regionservers

hd01.hz.idc.com

hd02.hz.idc.com

hd-node01.hz.idc.com

vim hbase-env.sh

# export HBASE_MANAGES_ZK=true

HBASE_MANAGES_ZK=false

启动Hbase服务

仅在hadoop-master上

# sudo /etc/init.d/hbase-master start

# sudo /etc/init.d/hbase-thrift start

# sudo /etc/init.d/hbase-rest start

仅在hadoop-node上

# sudo /etc/init.d/hbase-regionserver start

本文出自 “笔记本” 博客，请务必保留此出处http://zcnick.blog.51cto.com/1992444/1588973

Cloudera Hadoop 5.2 安装

声明：以上内容来自用户投稿及互联网公开渠道收集整理发布，本网站不拥有所有权，未作人工编辑处理，也不承担相关法律责任，若内容有误或涉及侵权可进行投诉：投诉/举报工作人员会在5个工作日内联系你，一经查实，本站将立刻删除涉嫌侵权内容。

联系
我们

首页 > 代码库 > Cloudera Hadoop 5.2 安装