mapreduce本地提交给集群中运行,单词统计案例
yarn高可用集群环境搭建同集群模式运行
集群模式运行:https://blog.csdn.net/weixin_43614067/article/details/108400938
本地模式运行:https://blog.csdn.net/weixin_43614067/article/details/108386389
修改Runner端
package com.bjsxt.wc;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import java.awt.*;
public class WCRunner {
public static void main(String[] args) throws Exception {
//创建配置对象
Configuration conf = new Configuration();
//默认文件系统的名称
conf.set("fs.defaultFS", "hdfs://node001:8020");*/
//本地运行,读取hdfs数据,并将数据提到hdfs
conf.set("fs.defaultFS", "hdfs://node001:8020");
/*本地集群运行map'mapreduce */
System.setProperty("HADOOP_USER_NAME","hadoop");
// 项目jar包地址。先编译打包,再运行main方法
conf.set("mapreduce.job.jar", "E:\\IDEAproject\\wordcount\\classes\\artifacts\\wordcount.jar");
conf.set("mapreduce.framework.name", "yarn");
//node002为ResourceManager的active节点
//命令yarn rmadmin -getServiceState rm1查看状态
conf.set("yarn.resourcemanager.address", "http://node002:8032");
conf.set("yarn.resourcemanager.scheduler.address","http://node002:8030");
//允许跨平台提交jar包
conf.set("mapreduce.app-submission.cross-platform", "true");
//创建Job对象
Job job = Job.getInstance(conf, "wordCount");
//设置mapper类
job.setMapperClass(WCMapper.class);
//设置 Reduce类
job.setReducerClass(WCReducer.class);
//设置运行job类
job.setJarByClass(WCRunner.class);
//设置map输出的key,value类型
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(IntWritable.class);
//设置reduce输出的key,value类型
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(IntWritable.class);
//设置输入路径金额输出路径
/* FileInputFormat.setInputPaths(job, new Path(args[0]));
FileOutputFormat.setOutputPath(job, new Path(args[1]));*/
FileInputFormat.setInputPaths(job, new Path("hdfs://node001:8020/wordcount/input"));
FileOutputFormat.setOutputPath(job, new Path("hdfs://node001:8020/wordcount/output"));
long startTime = System.currentTimeMillis();
try {
//传参是:是否显示程序运行状态及进度
boolean b = job.waitForCompletion(true);
if (b) {
System.out.println("单词统计完成!");
}
} finally {
// 结束的毫秒数
long endTime = System.currentTimeMillis();
System.out.println("Job<" + job.getJobName() + ">是否执行成功:" + job.isSuccessful() + "; 开始时间:" + startTime + "; 结束时间:" + endTime + "; 用时:" + (endTime - startTime) + "ms");
}
}
}
集群模式运行:https://blog.csdn.net/weixin_43614067/article/details/108400938
本地模式运行:https://blog.csdn.net/weixin_43614067/article/details/108386389