问题,在运行pig代码时候出现以下错误,强制类型转换的错误,但是在pig中并没有出现强制转换的语句(引用了UDF)。
AttemptID:attempt_1499336294438_15015_m_000001_0 Info:Error: java.lang.ClassCastException: java.lang.Integer cannot be cast to java.lang.String
at org.apache.pig.backend.hadoop.HDataType.getWritableComparableTypes(HDataType.java:106)
at org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigGenericMapReduce$Map.collect(PigGenericMapReduce.java:111)
at org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigGenericMapBase.runPipeline(PigGenericMapBase.java:284)
at org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigGenericMapBase.map(PigGenericMapBase.java:277)
at org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigGenericMapBase.map(PigGenericMapBase.java:64)
at org.apache.hadoop.mapreduce.Mapper.run(Mapper.java:145)
at org.apache.hadoop.mapred.MapTask.runNewMapper(MapTask.java:787)
at org.apache.hadoop.mapred.MapTask.run(MapTask.java:341)
at org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:163)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:415)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1671)
at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:158)
解决方案和问题原因
data_grp = FOREACH Data GENERATE FLATTEN(grp(type,lon,lat)) AS grpnumber:double;
这里调用UDF并且另命名为其他列,注意这里的数据类型要和UDF中返回的数据类型一致,如果不一致将会出现上面的那个错误。
package cn.sibat.taxi.sum;
import cn.sibat.taxi.sum.util.LocationUtil;
import org.apache.pig.EvalFunc;
import org.apache.pig.data.Tuple;
import org.apache.pig.data.TupleFactory;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
/**
* Created by User on 2017/9/20.
*/
public class Divide extends EvalFunc<Tuple> {
//粤B093YU,2016-12-31T23:02:20.000Z,113.95858,22.543633,1
static int groupnumber = 0;
static List<Tuple> store= new ArrayList<Tuple>();
static double sum =0;
static double final_sum = 0;
@Override
public Tuple exec(Tuple tuple) throws IOException {
TupleFactory tupleFactory = TupleFactory.getInstance();
Tuple out = tupleFactory.newTuple();
if (tuple.get(0) != null) {
int type = Integer.parseInt(tuple.get(0).toString());
if (store.size() == 0 || type == 1) {
store.add(tuple);
// out.append("-1");
// return out;
} else if (type == 0) {
groupnumber++;
for (int i = 0; i < store.size() - 1; i++) {
double distance = LocationUtil.distance(Double.parseDouble(store.get(i).get(1).toString()),
Double.parseDouble(store.get(i).get(2).toString()),
Double.parseDouble(store.get(i + 1).get(1).toString()),
Double.parseDouble(store.get(i + 1).get(2).toString()));
if (distance == 0 || distance > 1500) {//上下两点的距离等于0或大于1500米,过滤
continue;
}
//System.out.println("d:" + distance);
sum += distance;
}
final_sum=final_sum+sum;
out.append(final_sum);// final_sum 是double类型所以在pig中也是double类型
store.clear();
//final_sum=final_sum+sum;
sum = 0;
}
}
return out;
}
}