免密登录:https://blog.csdn.net/shenchengxinsunwei/article/details/84284110
架构信息
配置文件帮助文档:
http://hadoop.apache.org/docs/r2.7.6/hadoop-yarn/hadoop-yarn-site/ResourceManagerHA.html
yarn-site.xml配置。
<!-- 开启 yarn ha -->
<property>
<name>yarn.resourcemanager.ha.enabled</name>
<value>true</value>
</property>
<!-- 制定yarn ha名称 --->
<property>
<name>yarn.resourcemanager.cluster-id</name>
<value>cluster1</value>
</property>
<!--- 指定resourceManager名称(有多少个resourceManager就写几个) -->
<property>
<name>yarn.resourcemanager.ha.rm-ids</name>
<value>rm1,rm2</value>
</property>
<!-- 制定resourceManager 具体名称对应的服务器ip -->
<property>
<name>yarn.resourcemanager.hostname.rm1</name>
<value>master1</value>
</property>
<property>
<name>yarn.resourcemanager.hostname.rm2</name>
<value>master2</value>
</property>
<!-- 配置yarn WEB访问端口 -- >
<property>
<name>yarn.resourcemanager.webapp.address.rm1</name>
<value>master1:8088</value>
</property>
<property>
<name>yarn.resourcemanager.webapp.address.rm2</name>
<value>master2:8088</value>
</property>
<!-- 配置zookeeper集群信息 -->
<property>
<name>yarn.resourcemanager.zk-address</name>
<value>zk1:2181,zk2:2181,zk3:2181</value>
</property>
<!-- 开启resourceManager resart-->
<property>
<name>yarn.resourcemanager.recovery.enabled</name>
<value>true</value>
</property>
<!-- resourceManager 数据存储方式 可是zookeeper 也可以是hdfs,此处采用zookeeper ,zk地址上述已经配置此处不在配置-->
<property>
<name>yarn.resourcemanager.store.class</name>
<value>org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore</value>
</property>
<!-- 配置resourceManager 在zk的存储地址 -->
<property>
<name>yarn.resourcemanager.zk-state-store.parent-path</name>
<value>/rmstore</value>
</property>
<!-- 配置nodeManager restart 主要是保存containers 信息 -->
<!-- 开启nodeManager ha 重启 -->
<property>
<name>yarn.nodemanager.recovery.enabled</name>
<value>true</value>
</property>
<!-- yarn.nodemanager.recovery.dir 存储地址,默认值 -->
<!-- rpc 端口 默认0 不能访问,进行修改 ,官方提供 -->
<property>
<name>yarn.nodemanager.address</name>
<value>${yarn.nodemanager.hostname}:45454</value>
</property>
<property>
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce_shuffle</value>
</property>
<property>
<name>yarn.resourcemanager.hostname</name>
<value>slave1</value>
</property>
<!-- web application proxy安全 -->
<property>
<name>yarn.web-proxy.address</name>
<value>slave1:8888</value>
</property>
<!-- 开启日志 -->
<property>
<name>yarn.log-aggregation-enable</name>
<value>true</value>
</property>
<!--- 日志删除时间 -1禁用 -->
<property>
<name>yarn.log-aggregation.retain-seconds</name>
<value>604800</value>
</property>
<!-- yarn 内存 -->
<property>
<name>yarn.nodemanager.resource.memory-mb</name>
<value>4192</value>
</property>
<!-- yarn cpu 4核 -->
<property>
<name>yarn.nodemanager.resource.cpu-vcores</name>
<value>4</value>
</property>
1.启动整个集群 sbin/start-dfs.sh sbin/start-yarn.sh
2.在令一台resouceManager机器:启动resoruceManager sbin/yarn-daemon.sh start resourcemanager
3.问题:sbin/start-yarn.sh nodemanager 启动失败:java.lang.IllegalArgumentException: Cannot support recovery with an ephemeral server port. Check the setting of yarn.nodemanager.address
解决方式:参考文档:http://hadoop.apache.org/docs/r2.7.6/hadoop-yarn/hadoop-yarn-common/yarn-default.xml
<property>
<name>yarn.nodemanager.address</name>
<value>${yarn.nodemanager.hostname}:45454</value>
</property>