物理资源:CentOSA、CentOSB、CentOSC 64bit 内存2G
CentOSA:Zookepper、NameNode、DataNode、journalnode、zkfc、master(spark)、Worker(Spark)、Kafka
CentOSB:Zookepper、NameNode、DataNode、journalnode、zkfc、master(spark)、Worker(Spark)、Kafka
CentOSC:Zookepper、DataNode、journalnode、master(spark)、Worker(Spark)、Kafka
环境准备
- 主机名和IP映射
[root@CentOSX ~]# vi /etc/hosts
127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4
::1 localhost localhost.localdomain localhost6 localhost6.localdomain6
192.168.111.134 CentOSA
192.168.111.135 CentOSB
192.168.111.136 CentOSC
- 设置CentOS进程数和文件数(重启)
[root@CentOS ~]# vi /etc/security/limits.conf
* soft nofile 204800
* hard nofile 204800
* soft nproc 204800
* hard nproc 204800
[root@CentOS ~]# reboot
- 关闭防火墙
[root@CentOSX ~]# service iptables stop
iptables: Setting chains to policy ACCEPT: filter [ OK ]
iptables: Flushing firewall rules: [ OK ]
iptables: Unloading modules: [ OK ]
[root@CentOSX ~]# chkconfig iptables off
- 同步时钟
[root@CentOSX ~]# yum install -y ntp
[root@CentOSX ~]# ntpdate ntp.ubuntu.com
[root@CentOSX ~]# clock -w
- ssh免密认证
[root@CentOSX ~]# ssh-keygen -t rsa
Generating public/private rsa key pair.
Enter file in which to save the key (/root/.ssh/id_rsa):
Enter passphrase (empty for no passphrase):
Enter same passphrase again:
Your identification has been saved in /root/.ssh/id_rsa.
Your public key has been saved in /root/.ssh/id_rsa.pub.
The key fingerprint is:
2c:4b:ba:46:66:27:18:74:11:a7:96:f5:e5:96:97:d5 root@CentOSA
The key's randomart image is:
+--[ RSA 2048]----+
| ooo . .. |
| . .= . o . o E|
| . .+ . + o |
| .. . . . |
| o o S |
| . =o.o |
| +.o. |
| .. |
| .. |
+-----------------+
[root@CentOSX ~]# ssh-copy-id CentOSA
[root@CentOSX ~]# ssh-copy-id CentOSB
[root@CentOSX ~]# ssh-copy-id CentOSC
构建zookeeper集群
- 安装JDK 这里是jdk-8u191-linux-x64.rpm
[root@CentOSX ~]# rpm -ivh jdk-8u191-linux-x64.rpm
[root@CentOSX ~]# vi .bashrc
JAVA_HOME=/usr/java/latest
PATH=$PATH:$/bin
CLASSPATH=.
export JAVA_HOME
export PATH
export CLASSPATH
[root@CentOSX ~]# source .bashrc
[root@CentOSX ~]# java -version
java version "1.8.0_191"
Java(TM) SE Runtime Environment (build 1.8.0_191-b12)
Java HotSpot(TM) 64-Bit Server VM (build 25.191-b12, mixed mode)
- 安装配置zookeeper集群
[root@CentOSX ~]# tar -zxf zookeeper-3.4.6.tar.gz -C /usr/
[root@CentOSX ~]# mkdir /root/zkdata
[root@CentOSA ~]# echo 1 >> /root/zkdata/myid
[root@CentOSB ~]# echo 2 >> /root/zkdata/myid
[root@CentOSC ~]# echo 3 >> /root/zkdata/myid
[root@CentOSX ~]# vi /usr/zookeeper-3.4.6/conf/zoo.cfg
tickTime=2000
dataDir=/root/zkdata
clientPort=2181
initLimit=5
syncLimit=2
server.1=CentOSA:2887:3887
server.2=CentOSB:2887:3887
server.3=CentOSC:2887:3887
[root@CentOSX ~]# /usr/zookeeper-3.4.6/bin/zkServer.sh start zoo.cfg
[root@CentOSX ~]# /usr/zookeeper-3.4.6/bin/zkServer.sh status zoo.cfg
JMX enabled by default
Using config: /usr/zookeeper-3.4.6/bin/../conf/zoo.cfg
Mode: `follower|leader`
构建Kafka集群
配置与安装
[root@hadoopX ~]# tar -zxf kafka_2.11-2.2.0.tgz -C /usr
[root@hadoop2 ~]# vi /usr/kafka_2.11-2.2.0/config/server.properties
############################# Server Basics #############################
broker.id=[0|1|2]
############################# Socket Server Settings #############################
listeners=PLAINTEXT://CentOS[A|B|C]:9092
############################# Log Basics #############################
# A comma separated list of directories under which to store log files
log.dirs=/usr/kafka-logs
############################# Zookeeper #############################
zookeeper.connect=CentOSA:2181,CentOSB:2181,CentOSC:2181
启动服务
[root@CentOSX ~]# cd /usr/kafka_2.11-2.2.0/
[root@CentOSX kafka_2.11-2.2.0]# ./bin/kafka-server-start.sh -daemon config/server.properties
测试环境
- 创建topic01
[root@CentOSA kafka_2.11-2.2.0]# ./bin/kafka-topics.sh --zookeeper CentOSA:2181,CentOSB:2181,CentOSC:2181 --create --topic topic01 --partitions 3 --replication-factor 3
Created topic topic01.
这里partitions指定日志分区数目,replication-factor指定分区日志的副本因子。
- 消费者订阅topic01
[root@CentOSA kafka_2.11-2.2.0]# ./bin/kafka-console-consumer.sh
--bootstrap-server CentOSA:9092,CentOSB:9092,CentOSC:9092
--topic topic01
- 生产消息
[root@CentOSB kafka_2.11-2.2.0]# ./bin/kafka-console-producer.sh --broker-list CentOSA:9092,CentOSB:9092,CentOSC:9092 --topic topic01
> Hello Kafka
关闭Kafka服务
#!/bin/sh
SIGNAL=${SIGNAL:-TERM}
#PIDS=$(ps ax | grep -i 'kafka\.Kafka' | grep java | grep -v grep | awk '{print $1}')
PIDS=$(jps | grep 'Kafka' | awk '{print $1}')
if [ -z "$PIDS" ]; then
echo "No kafka server to stop"
exit 1
else
kill -s $SIGNAL $PIDS
fi
Hadoop NameNode HA
- 解压并配置HADOOP_HOME
[root@CentOSX ~]# tar -zxvf hadoop-2.9.2.tar.gz -C /usr/
[root@CentOSX ~]# vi .bashrc
HADOOP_HOME=/usr/hadoop-2.9.2
JAVA_HOME=/usr/java/latest
PATH=$PATH:$JAVA_HOME/bin:$HADOOP_HOME/bin:$HADOOP_HOME/sbin
CLASSPATH=.
export JAVA_HOME
export PATH
export CLASSPATH
export HADOOP_HOME
[root@CentOSX ~]# source .bashrc
- 配置core-site.xml
<!--配置Namenode服务ID-->
<property>
<name>fs.defaultFS</name>
<value>hdfs://mycluster</value>
</property>
<property>
<name>hadoop.tmp.dir</name>
<value>/usr/hadoop-2.9.2/hadoop-${user.name}</value>
</property>
<property>
<name>fs.trash.interval</name>
<value>30</value>
</property>
<!--配置机架脚本-->
<property>
<name>net.topology.script.file.name</name>
<value>/usr/hadoop-2.9.2/etc/hadoop/rack.sh</value>
</property>
<!--配置ZK服务信息-->
<property>
<name>ha.zookeeper.quorum</name>
<value>CentOSA:2181,CentOSB:2181,CentOSC:2181</value>
</property>
<!--配置SSH秘钥位置-->
<property>
<name>dfs.ha.fencing.methods</name>
<value>sshfence</value>
</property>
<property>
<name>dfs.ha.fencing.ssh.private-key-files</name>
<value>/root/.ssh/id_rsa</value>
</property>
- 配置机架脚本
[root@CentOSX ~]# touch /usr/hadoop-2.9.2/etc/hadoop/rack.sh
[root@CentOSX ~]# chmod u+x /usr/hadoop-2.9.2/etc/hadoop/rack.sh
[root@CentOSX ~]# vi /usr/hadoop-2.9.2/etc/hadoop/rack.sh
while [ $# -gt 0 ] ; do
nodeArg=$1
exec</usr/hadoop-2.9.2/etc/hadoop/topology.data
result=""
while read line ; do
ar=( $line )
if [ "${ar[0]}" = "$nodeArg" ] ; then
result="${ar[1]}"
fi
done
shift
if [ -z "$result" ] ; then
echo -n "/default-rack"
else
echo -n "$result "
fi
done
[root@CentOSX ~]# vi /usr/hadoop-2.9.2/etc/hadoop/topology.data
192.168.111.134 /rack01
192.168.111.135 /rack02
192.168.111.136 /rack03
- 配置hdfs-site.xml
<property>
<name>dfs.replication</name>
<value>3</value>
</property>
<!--开启自动故障转移-->
<property>
<name>dfs.ha.automatic-failover.enabled</name>
<value>true</value>
</property>
<!--解释core-site.xml内容-->
<property>
<name>dfs.nameservices</name>
<value>mycluster</value>
</property>
<property>
<name>dfs.ha.namenodes.mycluster</name>
<value>nn1,nn2</value>
</property>
<property>
<name>dfs.namenode.rpc-address.mycluster.nn1</name>
<value>CentOSA:9000</value>
</property>
<property>
<name>dfs.namenode.rpc-address.mycluster.nn2</name>
<value>CentOSB:9000</value>
</property>
<!--配置日志服务器的信息-->
<property>
<name>dfs.namenode.shared.edits.dir</name>
<value>qjournal://CentOSA:8485;CentOSB:8485;CentOSC:8485/mycluster</value>
</property>
<!--实现故障转切换的实现类-->
<property>
<name>dfs.client.failover.proxy.provider.mycluster</name>
<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
</property>
- 配置slaves
CentOSA
CentOSB
CentOSC
- 启动hdfs(集群初始化启动)
[root@CentOSX ~]# hadoop-daemon.sh start journalnode (等待10s钟)
[root@CentOSA ~]# hdfs namenode -format
[root@CentOSA ~]# hadoop-daemon.sh start namenode
[root@CentOSB ~]# hdfs namenode -bootstrapStandby
[root@CentOSB ~]# hadoop-daemon.sh start namenode
#注册Namenode信息到zookeeper中,只需要在CentOSA或者B上任意一台执行一下指令
[root@CentOSA|B ~]# hdfs zkfc -formatZK
[root@CentOSA ~]# hadoop-daemon.sh start zkfc
[root@CentOSB ~]# hadoop-daemon.sh start zkfc
[root@CentOSX ~]# hadoop-daemon.sh start datanode
构建spark Standalone
- 安装配置Spark
[root@CentOSX ~]# tar -zxf spark-2.4.3-bin-without-hadoop.tgz -C /usr/
[root@CentOSX ~]# mv /usr/spark-2.4.3-bin-without-hadoop/ /usr/spark-2.4.3
[root@CentOSX ~]# cd /usr/spark-2.4.3/conf/
[root@CentOS conf]# mv spark-env.sh.template spark-env.sh
[root@CentOS conf]# mv slaves.template slaves
[root@CentOS conf]# vi slaves
CentOSA
CentOSB
CentOSC
[root@CentOS conf]# vi spark-env.sh
SPARK_WORKER_CORES=4
SPARK_WORKER_MEMORY=2g
LD_LIBRARY_PATH=/usr/hadoop-2.9.2/lib/native
SPARK_DIST_CLASSPATH=$(hadoop classpath)
SPARK_DAEMON_JAVA_OPTS="-Dspark.deploy.recoveryMode=ZOOKEEPER -Dspark.deploy.zookeeper.url=CentOSA:2181,CentOSB:2181,CentOSC:2181 -Dspark.deploy.zookeeper.dir=/spark"
export SPARK_WORKER_CORES
export SPARK_WORKER_MEMORY
export LD_LIBRARY_PATH
export SPARK_DIST_CLASSPATH
export SPARK_DAEMON_JAVA_OPTS
- 启动spark服务
[root@CentOSX ~]# cd /usr/spark-2.4.3/
[root@CentOSA spark-2.4.3]# ./sbin/start-all.sh
[root@CentOSB spark-2.4.3]# ./sbin/start-master.sh
[root@CentOSC spark-2.4.3]# ./sbin/start-master.sh
- 测试Spark环境
[root@CentOSA spark-2.4.3]# ./bin/spark-shell --master spark://CentOSA:7077,CentOSB:7077,CentOSC:7077 --total-executor-cores 6
Setting default log level to "WARN".
To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).
Spark context Web UI available at http://CentOSA:4040
Spark context available as 'sc' (master = spark://CentOSA:7077,CentOSB:7077,CentOSC:7077, app id = app-20190708162400-0000).
Spark session available as 'spark'.
Welcome to
____ __
/ __/__ ___ _____/ /__
_\ \/ _ \/ _ `/ __/ '_/
/___/ .__/\_,_/_/ /_/\_\ version 2.4.3
/_/
Using Scala version 2.11.12 (Java HotSpot(TM) 64-Bit Server VM, Java 1.8.0_191)
Type in expressions to have them evaluated.
Type :help for more information.
scala>