spark-1.6.0-cdh5.15.0 standalone 模式安装
更多资源
-
github: https://github.com/opensourceteams/spark-scala-maven
-
csdn(汇总视频在线看): https://blog.csdn.net/thinktothings/article/details/84726769
前置条件
-
jdk ( 1.8.0_181)已安装
-
scala(2.10.7)已安装
-
hadoop(hadoop-2.6.0-cdh5.15.0)已安装,hdfs已启动
-
第三方jar
spark 依赖jar 文件配置,放置在 $SPARK_HOME/lib/*
parquet-hadoop-1.4.3.jar
jackson-databind-2.4.4.jar
jackson-annotations-2.4.4.jar
jackson-core-2.4.4.jar
jackson-module-scala_2.10-2.4.4.jar
<dependency>
<groupId>com.twitter</groupId>
<artifactId>parquet-hadoop</artifactId>
<version>1.4.3</version>
</dependency>
配置
环境变量配置
export JAVA_HOME=/opt/module/jdk/jdk1.8.0_191
export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
export SCALA_HOME=/opt/module/scala/scala-2.10.7
export HADOOP_HOME=/opt/module/bigdata/hadoop-2.6.0-cdh5.15.0
export SPARK_HOME=/opt/module/bigdata/spark-1.6.0-cdh5.15.0
export PATH=$JAVA_HOME/bin:$SCALA_HOME/bin:$SPARK_HOME/bin:$SPARK_HOME/sbin:$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$PATH
host配置
#/etc/hosts文件配置 IP到域名的配置
192.168.88.200 standalone.com standalone
spark-default.conf 配置
spark.master=spark://standalone.com:7077
spark.eventLog.enabled=true
spark.eventLog.dir=hdfs://standalone.com:9000/spark/log/eventLog
spark.serializer=org.apache.spark.serializer.KryoSerializer
spark.driver.memory=2g
#history
spark.history.fs.logDirectory=hdfs://standalone.com:9000/spark/log/historyEventLog
#The port to which the web interface of the history server binds.
spark.history.ui.port=18080
# The period at which information displayed by this history server is updated. Each update checks for any changes made to the event logs in persisted storage.
spark.history.fs.update.interval=10s
# The number of application UIs to retain. If this cap is exceeded, then the oldest applications will be removed.
spark.history.retainedApplications=50
spark.history.fs.cleaner.enabled=false
spark.history.fs.cleaner.interval=1d
spark.history.fs.cleaner.maxAge=7d
spark.history.ui.acls.enable=false
spark-env.sh 配置
export SPARK_DIST_CLASSPATH=${SPARK_HOME}/lib/*:$(${HADOOP_HOME}/bin/hadoop classpath)
SPARK_MASTER_IP=standalone.com
SPARK_MASTER_PORT=7077
SPARK_MASTER_WEBUI_PORT=8080
SPARK_WORKER_MEMORY=2g
SPARK_PRINT_LAUNCH_COMMAND=true #显示Main启动类执行的命令
salves配置
standalone.com
启动命令
启动 master 命令
start-master.sh
停止master命令
stop-master.sh
启动 worker 命令
start-slave.sh spark://standalone.com:7077
停止 worker 命令
stop-slave.sh
启动 history 命令
start-history-server.sh
停止 history 命令
stop-history-server.sh
启动 spark-shell 命令
spark-shell --master spark://standalone.com:7077
提交命令
spark-submit 命令
spark-submit \
--class com.opensource.bigdata.spark.standalone.RunTextFileMkString2 \
--master spark://standalone:7077 \
--executor-memory 1G \
--total-executor-cores 100 \
/root/temp/spark-scala-maven-1.0-SNAPSHOT.jar \
spark-submit \
--class com.opensource.bigdata.spark.standalone.RunTextFileMkString2 \
--master spark://standalone:7077 \
--deploy-mode client\
--executor-memory 1G \
--total-executor-cores 100 \
/root/temp/spark-scala-maven-1.0-SNAPSHOT.jar \