HDFS调用API
创建项目
pom.xml
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>Itcase</groupId>
<artifactId>HdfsApi</artifactId>
<version>1.0-SNAPSHOT</version>
<repositories>
<repository>
<id>cloudera</id>
<url>https://repository.cloudera.com/artifactory/cloudera-repos/</url>
</repository>
</repositories>
<dependencies>
<dependency>
<groupId>org.apache.Hadoop</groupId>
<artifactId>Hadoop-client</artifactId>
<version>2.6.0-mr1-cdh5.14.0</version>
</dependency>
<dependency>
<groupId>org.apache.Hadoop</groupId>
<artifactId>Hadoop-common</artifactId>
<version>2.6.0-cdh5.14.0</version>
</dependency>
<dependency>
<groupId>org.apache.Hadoop</groupId>
<artifactId>Hadoop-hdfs</artifactId>
<version>2.6.0-cdh5.14.0</version>
</dependency>
<dependency>
<groupId>org.apache.Hadoop</groupId>
<artifactId>Hadoop-mapreduce-client-core</artifactId>
<version>2.6.0-cdh5.14.0</version>
</dependency>
<!-- https://mvnrepository.com/artifact/junit/junit -->
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.11</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.testng</groupId>
<artifactId>testng</artifactId>
<version>RELEASE</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.0</version>
<configuration>
<source>1.8</source>
<target>1.8</target>
<encoding>UTF-8</encoding>
<!-- <verbal>true</verbal>-->
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>2.4.3</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<minimizeJar>true</minimizeJar>
</configuration>
</execution>
</executions>
</plugin>
<!-- <plugin>
<artifactId>maven-assembly-plugin </artifactId>
<configuration>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
<archive>
<manifest>
<mainClass>cn.itcast.Hadoop.db.DBToHdfs2</mainClass>
</manifest>
</archive>
</configuration>
<executions>
<execution>
<id>make-assembly</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>-->
</plugins>
</build>
</project>
代码
/**
* 创建文件夹
* @throws Exception
*/
@Test
public void mkdir01() throws Exception{
Configuration cf = new Configuration();
FileSystem fileSystem = FileSystem.get(new URI("hdfs://192.168.100.100:8020"), cf);
boolean mkdirs = fileSystem.mkdirs(new Path("/airpods/one"));
if (mkdirs){
System.out.println("创建成功");
}else{
System.out.println("创建失败");
}
}
/**
* 获取根目录下的所有文件夹路径
* @throws Exception
*/
@Test
public void listStatus() throws Exception{
Configuration cs = new Configuration();
FileSystem fs = FileSystem.get(new URI("hdfs://192.168.100.100:8020"), cs);
FileStatus[] fileStatuses = fs.listStatus(new Path("/"));
for (FileStatus fileStatus : fileStatuses) {
System.out.println(fileStatus.getPath().toString());
}
}
/**
* 修改文件夹名字
* @throws Exception
*/
@Test
public void rename() throws Exception{
Configuration configuration = new Configuration();
FileSystem fileSystem = FileSystem.get(new URI("hdfs://192.168.100.100:8020"), configuration);
Path path = new Path("/aaaaa");
Path path1 = new Path("/FiveA");
boolean rename = fileSystem.rename(path, path1);
String s = rename ? "成功" : "失败";
System.out.println(s);
}
/**
* 查看创建时间
* @throws Exception
*/
@Test
public void getTime() throws Exception{
Configuration configuration = new Configuration();
FileSystem fs = FileSystem.get(new URI("hdfs://192.168.100.100"), configuration);
FileStatus fileStatus = fs.getFileStatus(new Path("/FiveA"));
long modificationTime = fileStatus.getModificationTime();
System.out.println(modificationTime);
}
/**
- 删除文件夹
*/
@Test
public void deleteFile() throws Exception{
Configuration configuration = new Configuration();
FileSystem fileSystem = FileSystem.get(new URI("hdfs://192.168.100.100:8020"), configuration);
boolean delete = fileSystem.delete(new Path("/sum"), true);
System.out.println(delete);
}
/**
* 创建一个文件 并写入文件
* @throws Exception
*/
@Test
public void addFile()throws Exception{
Configuration configuration = new Configuration( );
FileSystem fileSystem = FileSystem.get(new URI("hdfs://192.168.100.100:8020"), configuration);
FSDataOutputStream outputStream = fileSystem.create(new Path("/airpods/one/后来.txt"));
byte[] bytes = "后来,终于在眼泪中明白".getBytes();
outputStream.write(bytes,0,bytes.length);
outputStream.close();
}
/**
* 上传本地文件
* @throws Exception
*/
@Test
public void put()throws Exception{
Configuration configuration = new Configuration();
FileSystem fileSystem = FileSystem.get(new URI("hdfs://192.168.100.100:8020"), configuration);
Path path = new Path("C:\\Users\\Desktop\\pomAPI.txt");
Path path1 = new Path("/aripods");
fileSystem.copyFromLocalFile(path,path1);
}
/**
* 判断该文件夹是否存在
* @throws Exception
*/
@Test
public void check()throws Exception{
Configuration configuration = new Configuration();
FileSystem fs = FileSystem.get(new URI("hdfs://192.168.100.100"), configuration);
boolean exists = fs.exists(new Path("/airpods"));
if (exists){
System.out.println("已存在");
}else{
System.out.println("不存在");
}
}
/**
* 读取本地文件内容之后在集群上创建新的文件吧内容写进去
* @throws Exception
*/
@Test
public void VimText() throws Exception{
String line=null;
Configuration configuration = new Configuration();
FileSystem fileSystem = FileSystem.get(configuration);
FSDataInputStream open = fileSystem.open(new Path("D:\\山丘.txt"));
FileSystem fileSystem1 = FileSystem.get(new URI("hdfs://192.168.100.100:8020"), configuration);
FSDataOutputStream fsDataOutputStream = fileSystem1.create(new Path("/airpods/sq.txt"));
BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(open));
while ((line=bufferedReader.readLine())!=null){
System.out.println(line);
fsDataOutputStream.write(line.getBytes(),0,line.getBytes().length);
}
}
强化练习
String uri=“hdfs://192.168.100.100:8020”;
//创建文件夹 — 如果该文件夹存在就删除 如果不存在就创建
@Test
public void mkdir() throws URISyntaxException, IOException {
Configuration configuration = new Configuration();
FileSystem fs = FileSystem.get(new URI(uri), configuration);
boolean exists = fs.exists(new Path("/opt"));
if (exists){
fs.delete(new Path(uri+"/opt"));
System.out.println("删除");
}else{
fs.mkdirs(new Path(uri+"/opt"));
System.out.println("创建");
}
}
// //获取所有的文件路径
@Test
public void list() throws URISyntaxException, IOException {
Configuration configuration = new Configuration();
FileSystem fileSystem = FileSystem.get(new URI(uri ), configuration);
FileStatus[] fileStatuses = fileSystem.listStatus(new Path("/"));
for (FileStatus fileStatus : fileStatuses) {
System.out.println(fileStatus.getPath().toString());
}
}
//创建一个新的文件并写入数据
@Test
public void write() throws Exception{
Configuration configuration = new Configuration();
FileSystem fileSystem = FileSystem.get(new URI(uri), configuration);
FSDataOutputStream fsDataOutputStream = fileSystem.create(new Path("/opt/1.txt"));
byte[] bytes = "李丽欣是一个大狼狗".getBytes();
fsDataOutputStream.write(bytes,0,bytes.length);
fsDataOutputStream.close();
}
//查看一个文件的内容
@Test
public void read()throws Exception{
String line=null;
Configuration configuration = new Configuration();
FileSystem fs = FileSystem.get(new URI(uri), configuration);
FSDataInputStream open = fs.open(new Path("/sq.txt"));
BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(open));
while ((line=bufferedReader.readLine())!=null){
System.out.println(line);
}
}
// 把本地文件的txt文件写入到集群上 集群的文件要新创建的 然后把本地的Txt文件内容写入到集群新的文件中 一边一边写
@Test
public void readANDwrit() throws Exception {
String line = null;
Configuration configuration = new Configuration();
FileSystem fileOpen = FileSystem.get(configuration);
FSDataInputStream open = fileOpen.open(new Path("D:\\山丘.txt"));
FileSystem fileCreate = FileSystem.get(new URI(uri), configuration);
FSDataOutputStream fsDataOutputStream = fileCreate.create(new Path("/sq.txt"));
BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(open));
while ((line=bufferedReader.readLine())!=null){
fsDataOutputStream.write(line.getBytes(),0,line.getBytes().length);
}
//修改集群上文件夹名字 并且读取你修改后的文件内容
@Test
public void rename()throws Exception{
Configuration configuration = new Configuration();
FileSystem fileSystem = FileSystem.get(new URI(uri), configuration);
boolean rename = fileSystem.rename(new Path("/sq.txt"), new Path("/山丘.txt"));
if (rename){
System.out.println("修改成功");
}else{
System.out.println("修改失败");
}
readThis("/山丘.txt");
}
// //创建一个文件夹 如果这个文件夹已存在那么就创建一个旧的名称+随机100以内的数
@Test
public void mkdirRound() throws URISyntaxException, IOException {
Configuration configuration = new Configuration();
FileSystem fs = FileSystem.get(new URI(uri), configuration);
if (fs.exists(new Path("/山丘"))){
System.out.println("已存在");
Random random = new Random();
int i = random.nextInt(100);
fs.mkdirs(new Path("/山丘"+i));
}else{
fs.mkdirs(new Path("/山丘"));
}
}
// //上传本地文件 如果已存在就把存在的给删除了
@Test
public void put() throws Exception{
Configuration configuration = new Configuration();
FileSystem fs = FileSystem.get(new URI(uri), configuration);
String Filename="aaa.txt";
//这是我本地的路径
Path bendi = new Path("D:\\"+Filename);
//这是HDFS的根目录
Path Hdfs = new Path("/");
//判断的是HDFS上是否存在该文件
boolean exists = fs.exists(new Path("/"+Filename));
if (exists){
//这是删除Hdfs上的文件而不是
fs.delete(new Path("/"+Filename));
System.out.println("已删除");
}else{
fs.copyFromLocalFile(bendi,Hdfs);
System.out.println("以上传到本地");
}
}
//判断文件是否存在如果存在就删除 如果不存在就创建
@Test
public void readThis(String name)throws Exception{
String line=null;
Configuration configuration = new Configuration();
FileSystem fs = FileSystem.get(new URI(uri), configuration);
FSDataInputStream open = fs.open(new Path(name));
BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(open));
while ((line=bufferedReader.readLine())!=null){
System.out.println(line);
}
}