hadoop(二)–java操作hdfs文件系统
一、简介
这里介绍使用java客户端操作hdfs文件系统。
二、 步骤
2.1 添加maven依赖
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>3.0.0</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<version>3.0.0</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-core</artifactId>
<version>3.0.0</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>3.0.0</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-annotations</artifactId>
<version>3.0.0</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-jobclient</artifactId>
<version>3.0.0</version>
</dependency>
2.2 hdfs操作
直接上代码,详情请查看代码注释
package com.dragon.study.hadoop;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.IOUtils;
import java.io.IOException;
import java.util.Arrays;
public class HadoopPaperMain {
public static void main(String[] args) throws Exception {
//创建连接
Configuration config = new Configuration();
config.set("fs.defaultFS", "hdfs://192.168.0.1:9000");
FileSystem fs = FileSystem.get(config);
//创建文件夹
Path newDirPath = new Path("/test1");
if(!fs.exists(newDirPath)){
fs.mkdirs(newDirPath);
}
//查看指定目录下的文件
Path listDirPath = new Path("/test3");
FileStatus[] fileStatuses = fs.listStatus(listDirPath);
Arrays.stream(fileStatuses).forEach(fileStatus -> System.out.println(fileStatus.getPath().toString()));
//写文件
Path newFilePath = new Path("/test3/t1.txt");
FSDataOutputStream fsout = fs.create(newFilePath);
fsout.write("好好学习,天天向上".getBytes("utf8"));
// fsout.writeChars("study hard ");
fsout.flush();
fsout.close();
//读文件
Path readFilePath = new Path("/test2/t2.txt");
FSDataInputStream fsin = fs.open(readFilePath);
IOUtils.copyBytes(fsin.getWrappedStream(), System.out,4096);
//查看文件位置信息
Path locationFilePath = new Path("/test3/t1.txt");
FileStatus fileStatus = fs.getFileStatus(locationFilePath);
BlockLocation[] locations = fs.getFileBlockLocations(locationFilePath, 0,fileStatus.getLen());
Arrays.stream(locations).forEach(location -> {
try {
System.out.println(Arrays.toString(location.getHosts()));
} catch (IOException e) {
e.printStackTrace();
}
});
//删除文件或文件夹
Path delPath = new Path("/test1");
fs.delete(delPath, true);
//上传文件
Path srcPath = new Path("E:/tmp/t2.txt");
Path dstPath = new Path("/test2/");
fs.copyFromLocalFile(srcPath,dstPath);
fs.close();
}
}