上一节已经说道持久化IP和安装JDK
这一节继续搭建Hadoop分布式开发环境
一、安装Hadoop包
1、使用下载的hadoop-2.7.2.tar.gz,上传到虚拟机的/usr/local目录下。
2、将hadoop包进行解压缩:
tar -zxvf hadoop-2.7.2.tar.gz
3、对hadoop目录进行重命名:
mv hadoop-2.7.2 hadoop
4、配置hadoop相关环境变量
vi ~/.bashrc 进入编辑器
export HADOOP_HOME=/usr/local/hadoop --添加
export PATH=$HADOOP_HOME/bin:$HADOOP_HOME/sbin --追加
保存后执行
source ~/.bashrc
5、创建/usr/local/hadoop/dfs
,/usr/local/hadoop/dfs/name
,/usr/local/hadoop/dfs/data
目录
mkdir /usr/local/hadoop/dfs
mkdir /usr/local/hadoop/dfs/name
mkdir /usr/local/hadoop/dfs/data
6、创建tmp目录
mkdir /usr/local/hadoop/tmp
二、修改相关配置文件
文件位于/usr/local/hadoop/etc/hadoop目录下(修改使用vi)
cd /usr/local/hadoop/etc/hadoop
core-site.xml
fs.defaultFS
hdfs://master:9000
io.file.buffer.size
131072
hadoop.tmp.dir
file:/usr/local/hadoop/tmp
Abase for other temporary directories.
hdfs-site.xml
<configuration>
<property>
<name>fs.defaultFS</name>
<value>hdfs://master:9000</value>
</property>
<property>
<name>io.file.buffer.size</name>
<value>131072</value>
</property>
<property>
<name>hadoop.tmp.dir</name>
<value>file:/usr/local/hadoop/tmp</value>
<description>Abase for other temporary directories.</description>
</property>
</configuration>
- mapred-site.xml
配置该文件时需要将mapred-site.xml.template重命名为mapred-site.xml
mv mapred-site.xml.template mapred-site.xml --重命名
<configuration>
<property>
<name>mapreduce.framework.name</name>
<value>yarn</value>
</property>
<property>
<name>mapreduce.jobhistory.address</name>
<value>master:10020</value>
</property>
<property>
<name>mapreduce.jobhistory.webapp.address</name>
<value>master:19888</value>
</property>
</configuration>
- yarn-site.xml
<configuration>
<property>
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce_shuffle</value>
</property>
<property>
<name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>
<value>org.apache.hadoop.mapred.ShuffleHandler</value>
</property>
<property>
<name>yarn.resourcemanager.address</name>
<value>master:8032</value>
</property>
<property>
<name>yarn.resourcemanager.scheduler.address</name>
<value>master:8030</value>
</property>
<property>
<name>yarn.resourcemanager.resource-tracker.address</name>
<value>master:8031</value>
</property>
<property>
<name>yarn.resourcemanager.admin.address</name>
<value>master:8033</value>
</property>
<property>
<name>yarn.resourcemanager.webapp.address</name>
<value>master:8088</value>
</property>
<property>
<name>yarn.resourcemanager.hostname</name>
<value>192.168.1.151</value>
</property>
</configuration>
- slaves文件
slave1
slave2