package com.sort; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; import org.apache.hadoop.util.GenericOptionsParser; public class Entry { @SuppressWarnings("deprecation") public static void main(String[] args) throws Exception { Configuration conf = new Configuration(); String[] otherarg = new GenericOptionsParser(conf, args).getRemainingArgs(); if (otherarg.length != 2) { System.out.println("error!"); System.exit(2); } Job job = new Job(conf, "number sort"); job.setJarByClass(Entry.class); job.setMapperClass(Mappersort.class); job.setReducerClass(Reducersort.class); job.setOutputKeyClass(IntWritable.class); job.setOutputValueClass(IntWritable.class); FileInputFormat.addInputPath(job, new Path(otherarg[0])); FileOutputFormat.setOutputPath(job, new Path(otherarg[1])); System.exit(job.waitForCompletion(true) ? 0 : 1); } } package com.sort; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Mapper; import java.io.*; public class Mappersort extends Mapper<Object, Text, IntWritable, IntWritable> { private static IntWritable num=new IntWritable(); public void map(Object key, Text value, Context context) throws IOException, InterruptedException { String []line=value.toString().split("\n"); num.set(Integer.parseInt(line[0])); context.write(num,new IntWritable(1)); } } package com.sort; import java.io.IOException; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.mapreduce.Reducer; public class Reducersort extends Reducer<IntWritable, IntWritable, IntWritable, IntWritable> { private static IntWritable num = new IntWritable(1); public void reduce(IntWritable key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException { for (IntWritable val : values) { context.write(num, key); num = new IntWritable(num.get() + 1); } } }
设计思路
熟悉MapReduce过程会很快想到在MapReduce过程中就有排序,是否可以利用这个默认的排序,而不需要自己再实现具体的排序呢?答案是肯定的。
但是在使用之前首先需要了解它的默认排序规则。它是按照key值进行排序的,如果key为封装int的IntWritable类型,那么MapReduce按照数字大小对key排序,如果key为封装为String的Text类型,那么MapReduce按照字典顺序对字符串排序。
了解了这个细节,我们就知道应该使用封装int的IntWritable型数据结构了。也就是在map中将读入的数据转化成IntWritable型,然后作为key值输出(value任意)。reduce拿到<key,value-list>之后,将输入的key作为value输出,并根据value-list中元素的个数决定输出的次数。输出的key(即代码中的linenum)是一个全局变量,它统计当前key的位次。需要注意的是这个程序中没有配置Combiner,也就是在MapReduce过程中不使用Combiner。这主要是因为使用map和reduce就已经能够完成任务了。
Map接收输入key为每行偏移量,value为该行数据(字符型),将value转换为整型(Integer.parseInt(sting)),做为map输出的key值,value值写1;Reduce接收的输入key值为需要排序的整数,value-list为该整数出现的次数,reduce输出key值为全局序号(由各个key值的value-list决定),value值为排序完成的整数。