版权声明:本文为博主原创文章,未经博主允许不得转载。 https://blog.csdn.net/qq_26437925/article/details/80017146
图书《hadoop 技术内幕: 深入解析Hadoop Common和HDFS》
windows环境
- jdk
- ant
- Cygwin
- hadoop-1.0.0.tar.gz
- eclipse
解压hadoop-1.0.0.tar.gz,Cygwin进入目录后执行 ant eclipse
- 错误1
ant-eclipse-1.0.bin.tar.bz2下载可能超时,直接本地下载后放到hadoop-1.0.0/build目录下,然后注释掉hadoop-1.0.0/build.xml中的下面两行
<get src="http://downloads.sourceforge.net/project/ant-eclipse/ant-eclipse/1.0/ant-eclipse-1.0.bin.tar.bz2"
dest="${build.dir}/ant-eclipse-1.0.bin.tar.bz2" usetimestamp="false" />
- 错误2
BUILD FAILED
D:\hadoop\hadoop-1.0.0\build.xml:2295: D:\hadoop\hadoop-1.0.0\.eclipse.templates does not exist.
新建一个.eclipse.templates目录
build成功后,直接导入eclipse
- 错误3
TestRandomAlgorithm类“import com.sun.tools.javac.code.Attribute.Array;”提示找不到,可以将其注释或者将jdk目录下/lib/tools.jar加到工程中
最后源码成功导入
Hadoop 配置
配置文件采用xml格式
<configuration>
<property>
<name>属性名称</name>
<value>属性值</value>
<description>属性描述</description>
</property>
</configuration>
Hadoop 使用 org.apache.hadoop.conf.Configuration 处理配置信息
Hadoop源码自带了很多Test,可以直接运行这些测试例子,
例如 Configuration的测试例子如下
可以用git进行源码修改控制
下面是自己仿照写的ConfigureTest,
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.conf;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.DataInputStream;
import java.io.ByteArrayOutputStream;
import java.io.ByteArrayInputStream;
import java.io.DataOutputStream;
import java.io.StringWriter;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map;
import java.util.Random;
import junit.framework.TestCase;
import org.apache.hadoop.fs.Path;
import org.codehaus.jackson.map.ObjectMapper;
public class MyTestConfiguration extends TestCase {
private Configuration conf;
final static String CONFIG = new File("./my-test-config.xml").getAbsolutePath();
final static Random RAN = new Random();
@Override
protected void setUp() throws Exception {
super.setUp();
conf = new Configuration();
}
@Override
protected void tearDown() throws Exception {
super.tearDown();
//new File(CONFIG).delete();
}
private void startConfig() throws IOException{
out.write("<?xml version=\"1.0\"?>\n");
out.write("<configuration>\n");
}
private void endConfig() throws IOException{
out.write("</configuration>\n");
out.close();
}
private void addInclude(String filename) throws IOException{
out.write("<xi:include href=\"" + filename + "\" xmlns:xi=\"http://www.w3.org/2001/XInclude\" />\n ");
}
public void testWriteRead() throws Exception {
tearDown();
System.out.println("==============");
// 写入xml
out=new BufferedWriter(new FileWriter(CONFIG));
startConfig();
appendProperty("a","aaa");
appendProperty("b","bbb");
endConfig();
// 读取判断
Path fileResource = new Path(CONFIG);
conf.addResource(fileResource);
String val_a = conf.get("a");
String val_b = conf.get("b");
System.out.println(val_a);
System.out.println(val_b);
System.out.println("==============");
tearDown();
}
public static void assertEq(Object a, Object b) {
System.out.println("assertEq: " + a + ", " + b);
assertEquals(a, b);
}
static class Prop {
String name;
String val;
String expectEval;
}
final String UNSPEC = null;
ArrayList<Prop> props = new ArrayList<Prop>();
void declareProperty(String name, String val, String expectEval)
throws IOException {
declareProperty(name, val, expectEval, false);
}
void declareProperty(String name, String val, String expectEval,
boolean isFinal)
throws IOException {
appendProperty(name, val, isFinal);
Prop p = new Prop();
p.name = name;
p.val = val;
p.expectEval = expectEval;
props.add(p);
}
void appendProperty(String name, String val) throws IOException {
appendProperty(name, val, false);
}
void appendProperty(String name, String val, boolean isFinal)
throws IOException {
out.write("<property>");
out.write("<name>");
out.write(name);
out.write("</name>");
out.write("<value>");
out.write(val);
out.write("</value>");
if (isFinal) {
out.write("<final>true</final>");
}
out.write("</property>\n");
}
BufferedWriter out;
}
运行截图