HA
Redis HA
(1)复制sentinel.conf文件到conf文件夹下cp sentinel.conf /usr/local/redis/conf/
(2)修改sentinel.conf文件
sentinel monitor mymaster 192.168.138.130 6379 1
(3)启动Redis集群
redis-server conf/redis6379.conf
redis-server conf/redis6380.conf
redis-server conf/redis6381.conf
(4)启动Redis哨兵
redis-sentinel conf/sentinel.conf
(5)查看Redis进程,并杀死
ps -ef | grep redis
kill -9 2634
(6)哨兵就会重新选取新的主节点
Hadoop HA
(1)修改hadoop下的core-site.xml文件<configuration>
<!-- 指定HDFS的nameservice为hacluster -->
<property>
<name>fs.defaultFS</name>
<value>hdfs://hacluster</value>
</property>
<!-- 指定zookeeper地址 -->
<property>
<name>ha.zookeeper.quorum</name>
<value>hadoop1:2181,hadoop2:2181,hadoop3:2181</value>
</property>
<!-- 配置HDFS数据块和元数据保存的目录 -->
<property>
<name>hadoop.tmp.dir</name>
<value>/usr/local/hadoop/tmp</value>
</property>
</configuration>
(2)修改hadoop下的hdfs-site.xml文件
<configuration>
<!-- 指定副本数 -->
<property>
<name>dfs.replication</name>
<value>2</value>
</property>
<!-- 配置namenode和datanode的工作目录-数据存储目录 -->
<property>
<name>dfs.namenode.name.dir</name>
<value>/usr/local/hadoop/dfs/name</value>
</property>
<property>
<name>dfs.datanode.data.dir</name>
<value>/usr/local/hadoop/dfs/data</value>
</property>
<!-- 启用webhdfs -->
<property>
<name>dfs.webhdfs.enabled</name>
<value>true</value>
</property>
<!-- 指定HDFS的nameservice为hacluster -->
<property>
<name>dfs.nameservices</name>
<value>hacluster</value>
</property>
<!-- hacluster有两个NamoNode -->
<property>
<name>dfs.ha.namenodes.hacluster</name>
<value>nn1,nn2</value>
</property>
<!-- nn1的RPC通信地址 -->
<property>
<name>dfs.namenode.rpc-address.hacluster.nn1</name>
<value>hadoop1:9000</value>
</property>
<!-- nn1的http通信地址 -->
<property>
<name>dfs.namenode.http-address.hacluster.nn1</name>
<value>hadoop1:50070</value>
</property>
<!-- nn2的RPC通信地址 -->
<property>
<name>dfs.namenode.rpc-address.hacluster.nn2</name>
<value>hadoop2:9000</value>
</property>
<!-- nn2的http通信地址 -->
<property>
<name>dfs.namenode.http-address.hacluster.nn2</name>
<value>hadoop2:50070</value>
</property>
<!-- 指定NameNode的edits元数据的共享存储位置,journalId推荐使用nameservice,默认端口号是:8485 -->
<property>
<name>dfs.namenode.shared.edits.dir</name>
<value>qjournal://hadoop1:8485;hadoop2:8485;hadoop3:8485/hacluster</value>
</property>
<!-- 指定JournalNode在本地磁盘存放数据的位置 -->
<property>
<name>dfs.journalnode.edits.dir</name>
<value>/usr/local/hadoop/journal</value>
</property>
<!-- 配置失败自动切换实现方式 -->
<property>
<name>dfs.client.failover.proxy.provider.hacluster</name>
<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
</property>
<!-- 配置隔离机制方法,多个机制用换行分割,即每个机制暂用一行 -->
<property>
<name>dfs.ha.fencing.methods</name>
<value>
sshfence
shell(/bin/true)
</value>
</property>
<!-- 使用sshfence隔离机制时需要ssh免登陆 -->
<property>
<name>dfs.ha.fencing.ssh.private-key-files</name>
<value>/home/vagrant/.ssh/id_rsa</value>
</property>
<!-- 开启NameNode失败自动切换 -->
<property>
<name>dfs.ha.automatic-failover.enabled</name>
<value>true</value>
</property>
</configuration>
(3)修改mapred-site.xml文件
<configuration>
<!--配置MR程序运行的框架-->
<property>
<name>mapreduce.framework.name</name>
<value>yarn</value>
</property>
</configuration>
(4)修改yarn-site.xml文件
<configuration>
<!-- 开启RM高可用 -->
<property>
<name>yarn.resourcemanager.ha.enabled</name>
<value>true</value>
</property>
<!-- 指定RM的cluster id -->
<property>
<name>yarn.resourcemanager.cluster-id</name>
<value>yrc</value>
</property>
<!-- 指定RM的名字 -->
<property>
<name>yarn.resourcemanager.ha.rm-ids</name>
<value>rm1,rm2</value>
</property>
<!-- 分别指定RM的地址 -->
<property>
<name>yarn.resourcemanager.hostname.rm1</name>
<value>hadoop1</value>
</property>
<property>
<name>yarn.resourcemanager.hostname.rm2</name>
<value>hadoop2</value>
</property>
<!-- 指定zk集群地址 -->
<property>
<name>yarn.resourcemanager.zk-address</name>
<value>hadoop1:2181,hadoop2:2181,hadoop3:2181</value>
</property>
<!--NodeManager执行MR任务的方式是Shuffle洗牌-->
<property>
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce_shuffle</value>
</property>
<property>
<name>yarn.log-aggregation-enable</name>
<value>true</value>
</property>
<!-- 启用自动恢复 -->
<property>
<name>yarn.resourcemanager.recovery.enabled</name>
<value>true</value>
</property>
<!-- 制定resourcemanager的状态信息存储在zookeeper集群上 -->
<property>
<name>yarn.resourcemanager.store.class</name>
<value>org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore</value>
</property>
<property>
<name>yarn.nodemanager.resource.memory-mb</name>
<value>32768</value>
</property>
<property>
<name>yarn.scheduler.minimum-allocation-mb</name>
<value>4096</value>
</property>
<property>
<name>yarn.nodemanager.resource.cpu-vcores</name>
<value>24</value>
</property>
<property>
<name>yarn.nodemanager.remote-app-log-dir</name>
<value>/usr/local/hadoop/tmp/yarn-logs</value>
</property>
</configuration>
(5)修改slaves文件
Hadoop1
Hadoop2
Hadoop3
(6)分发配置文件到其他服务器
scp -r hadoop hadoop2:$PWD
scp -r hadoop hadoop3:$PWD
(7)启动各服务器的zookeeper
扫描二维码关注公众号,回复:
10829887 查看本文章
![](/qrcode.jpg)
zkServer.sh start
(8)启动各服务器的journalnode
hadoop-daemon.sh start journalnode
(9)清除datanode和namenode的文件夹
rm -rf /usr/local/hadoop/dfs/name/*
rm -rf /usr/local/hadoop/dfs/data/*
(10)格式化hadoop1的namenode
hadoop namenode -format
(10)将hadoop1的元信息发送到hadoop2上
scp -r /usr/local/hadoop/tmp/dfs hadoop2:/usr/local/hadoop/tmp/
# 或在另一个namenode上执行
hadoop namenode -bootstrapStandby
(11)格式化hadoop1和hadoop2的zkfc
hdfs zkfc -formatZK
(12)启动Hadoop集群
start-all.sh
(13)杀掉hadoop1的进程
[root@Hadoop1 tmp]# jps
2337 QuorumPeerMain
11827 NameNode
12275 DFSZKFailoverController
12533 NodeManager
11239 JournalNode
13607 Jps
12426 ResourceManager
[root@Hadoop1 tmp]# kill -9 11827
Hbase HA
(1)修改hbase-env.shexport HBASE_MANAGES_ZK=false
(2)修改hbase-site.xml文件
<configuration>
<!-- 设置NameNode所在位置,通过rootdir设置,也就是设置HDFS中存放的路径 -->
<property>
<name>hbase.rootdir</name>
<value>hdfs://hacluster/hbase</value>
</property>
<!-- 是否开启集群 -->
<property>
<name>hbase.cluster.distributed</name>
<value>true</value>
</property>
<!-- zookeeper集群的位置 -->
<property>
<name>hbase.zookeeper.quorum</name>
<value>hadoop1,hadoop2,hadoop3</value>
</property>
<property>
<name>hbase.zookeeper.property.clientPort</name>
<value>2181</value>
</property>
<property>
<name>zookeeper.session.timeout</name>
<value>120000</value>
</property>
<property>
<name>hbase.zookeeper.property.tickTime</name>
<value>6000</value>
</property>
</configuration>
(3)修改regionservers文件
hadoop1
hadoop2
hadoop3
(4)启动hbase集群
# hadoop1
hbase-daemon.sh start master
hbase-daemon.sh start regionserver
# hadoop2
hbase-daemon.sh start master
hbase-daemon.sh start regionserver
# hadoop3
hbase-daemon.sh start regionserver
(5)杀掉hadoop1的进程
[root@Hadoop1 logs]# jps
25648 NodeManager
25523 ResourceManager
25029 DataNode
35717 Jps
2215 QuorumPeerMain
26135 NameNode
34936 HRegionServer
25226 JournalNode
25435 DFSZKFailoverController
34510 HMaster
[root@Hadoop1 logs]# kill -9 34510