HDFS之JAVA API

===============================================================================

一.创建目录

public static void mkdir() throws IOException {
        //实例化配置信息对象，简而言之Configuration类用于连接Windows中的idea软件与Linux中的hdfs
        Configuration configuration = new Configuration();
        //利用对象.set连接hdfs,niit为我的主机名，具体根据自己的改，9000不变
        configuration.set("fs.defaultFS","hdfs://niit:9000");
        //实例化分布式文件系统对象：FileSystem（父类）
        FileSystem fileSystem = FileSystem.get(configuration);
        //利用FileSystem对象的来实现目录的创建：mkdir
        fileSystem.mkdirs(new Path("/new2"));
        //并行执行不利，用完要关掉
        fileSystem.close();
        //提示
        System.out.println("创建成功~");
    }

二.删除目录

public static void deleteDir() throws IOException {
        Configuration configuration = new Configuration();
        configuration.set("fs.defaultFS","hdfs://niit:9000");
        FileSystem fileSystem = FileSystem.get(configuration);
        boolean isDel = fileSystem.deleteOnExit(new Path("/new2"));
        if (isDel){
            System.out.println("删除成功~");
            return;
        }
        System.out.println("删除失败!");
    }

三.上传本地文件至hdfs

public static void testCopyFromLocalFile() throws IOException, InterruptedException, URISyntaxException {
        // 1 获取文件系统
        Configuration configuration = new Configuration();
        configuration.set("dfs.replication", "2");//副本数2
        FileSystem fs = FileSystem.get(new URI("hdfs://niit:9000"), configuration, "atguigu");
        // 2 上传文件
        fs.copyFromLocalFile(new Path("D:\\College job file\\docx\\12.txt"), new Path("/12.txt"));
        // 3 关闭资源
        fs.close();
        System.out.println("上传成功~");
    }

四.文件重命名

public static void testRename() throws IOException, InterruptedException, URISyntaxException{
        // 1 获取文件系统
        Configuration configuration = new Configuration();
        FileSystem fs = FileSystem.get(new URI("hdfs://niit:9000"), configuration, "atguigu");
        // 2 修改文件名称
        fs.rename(new Path("/hello.txt"), new Path("/hello01.txt"));
        System.out.println("更改成功~");
        // 3 关闭资源
        fs.close();
    }

五.文件从hdfs下载到本机

   /**
     * 文件下载
     * 1.明确目的地：Configuration
     * 2.明确本地位置：FileOutputStream
     * 3.找到目标地址的文件：FileSystem
     * 4.读取文件：FSDataInputStream
     * 5.将目标文件复制到本机：IOUtils
     * */
    public static void downloadFromHDFS() throws IOException {
        Configuration configuration = new Configuration();
        configuration.set("fs.defaultFS","hdfs://niit:9000");
        FileSystem fileSystem = FileSystem.get(configuration);
        FileOutputStream outputStream = new FileOutputStream(new File("D:\\downloadFromHDFS\\11.txt"));
        FSDataInputStream inputStream = fileSystem.open(new Path("/18-Batch2/11.txt"));
        IOUtils.copyBytes(inputStream, outputStream,configuration);
        fileSystem.close();
        System.out.println("下载成功~");
    }

六.文件从hdfs下载到本机

public static void listFiles() throws IOException {
        Configuration configuration = new Configuration();
        configuration.set("fs.defaultFS","hdfs://niit:9000");
        FileSystem fileSystem = FileSystem.get(configuration);
        //调用方法查看文件详情
        RemoteIterator<LocatedFileStatus> iterator = fileSystem.listFiles(new Path("/18-Batch2"),true);
        //Iterator是一个迭代器,一个接口（数组，list），<LocatedFileStatus>用于封装文件的属性信息，继承了FileStatus这个类
        while (iterator.hasNext()){
            LocatedFileStatus locatedFileStatus = iterator.next();
            System.out.println("BlockSize:" + locatedFileStatus.getBlockSize());
            System.out.println("Owner:" + locatedFileStatus.getOwner());
            System.out.println("Group:" + locatedFileStatus.getGroup());
            System.out.println("UpdateTime:" + locatedFileStatus.getModificationTime());
            System.out.println("Replication:" + locatedFileStatus.getReplication());
            BlockLocation[] blockLocations = locatedFileStatus.getBlockLocations();
            for (BlockLocation block: blockLocations){
                System.out.println("len : " + block.getLength());
                String[] blockNames = block.getNames();
                for (String name: blockNames){
                    System.out.println("block name : " + name);
                }
            }
        }
        fileSystem.close();
    }

七.判断文件文件夹

public static void testListStatus() throws IOException, InterruptedException, URISyntaxException {
        // 1获取文件系统
        Configuration configuration = new Configuration();
        FileSystem fs = FileSystem.get(new URI("hdfs://niit:9000"), configuration, "atguigu");
        // 2 判断是否是文件还是文件夹
        FileStatus[] listStatus = fs.listStatus(new Path("/"));
        for (FileStatus fileStatus : listStatus) {
            // 如果是文件file: ，如果文件夹directory:
            if (fileStatus.isFile()) {
                System.out.println("file:"+fileStatus.getPath().getName());
            }else {
                System.out.println("directory:"+ fileStatus.getPath().getName());
            }
        }
    }

总体代码

package com.demo;

import com.google.inject.internal.util.$SourceProvider;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.IOUtils;
import org.graalvm.compiler.core.common.type.ArithmeticOpTable;
import org.junit.Test;

import javax.security.auth.login.AppConfigurationEntry;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.Iterator;
import java.util.List;
import java.util.Properties;

public class HdfsTest {
    public static void main(String[] args) throws IOException, URISyntaxException, InterruptedException {
        //mkdir();
        //deleteDir();
        //testCopyFromLocalFile();
        //testRename();
        //downloadFromHDFS();
        //listFiles();
        //testListStatus();
    }
    //创建目录
    public static void mkdir() throws IOException {
        Configuration configuration = new Configuration();
        configuration.set("fs.defaultFS","hdfs://niit:9000");
        FileSystem fileSystem = FileSystem.get(configuration);
        fileSystem.mkdirs(new Path("/new2"));
        fileSystem.close();
        System.out.println("创建成功~");
    }


    //删除目录
    public static void deleteDir() throws IOException {
        Configuration configuration = new Configuration();
        configuration.set("fs.defaultFS","hdfs://niit:9000");
        FileSystem fileSystem = FileSystem.get(configuration);
        boolean isDel = fileSystem.deleteOnExit(new Path("/new2"));
        if (isDel){
            System.out.println("删除成功~");
            return;
        }
        System.out.println("删除失败!");
    }


    //上传文件
    public static void testCopyFromLocalFile() throws IOException, InterruptedException, URISyntaxException {
        Configuration configuration = new Configuration();
        configuration.set("dfs.replication", "2");
        FileSystem fs = FileSystem.get(new URI("hdfs://niit:9000"), configuration, "atguigu");
        fs.copyFromLocalFile(new Path("D:\\College job file\\docx\\12.txt"), new Path("/12.txt"));
        fs.close();
        System.out.println("上传成功~");
    }


    //重命名
    public static void testRename() throws IOException, InterruptedException, URISyntaxException{
        Configuration configuration = new Configuration();
        FileSystem fs = FileSystem.get(new URI("hdfs://niit:9000"), configuration, "atguigu");
        fs.rename(new Path("/hello.txt"), new Path("/hello01.txt"));
        System.out.println("更改成功~");
        fs.close();
    }


  
     //文件下载
    public static void downloadFromHDFS() throws IOException {
        Configuration configuration = new Configuration();
        configuration.set("fs.defaultFS","hdfs://niit:9000");
        FileSystem fileSystem = FileSystem.get(configuration);
        FileOutputStream outputStream = new FileOutputStream(new File("D:\\downloadFromHDFS\\11.txt"));
        FSDataInputStream inputStream = fileSystem.open(new Path("/18-Batch2/11.txt"));
        IOUtils.copyBytes(inputStream, outputStream,configuration);
        fileSystem.close();
        System.out.println("下载成功~");
    }


    //查看文件详情
    public static void listFiles() throws IOException {
        Configuration configuration = new Configuration();
        configuration.set("fs.defaultFS","hdfs://niit:9000");
        FileSystem fileSystem = FileSystem.get(configuration);
        RemoteIterator<LocatedFileStatus> iterator = fileSystem.listFiles(new Path("/18-Batch2"),true);
        while (iterator.hasNext()){
            LocatedFileStatus locatedFileStatus = iterator.next();
            System.out.println("BlockSize:" + locatedFileStatus.getBlockSize());
            System.out.println("Owner:" + locatedFileStatus.getOwner());
            System.out.println("Group:" + locatedFileStatus.getGroup());
            System.out.println("UpdateTime:" + locatedFileStatus.getModificationTime());
            System.out.println("Replication:" + locatedFileStatus.getReplication());
            BlockLocation[] blockLocations = locatedFileStatus.getBlockLocations();
            for (BlockLocation block: blockLocations){
                System.out.println("len : " + block.getLength());
                String[] blockNames = block.getNames();
                for (String name: blockNames){
                    System.out.println("block name : " + name);
                }
            }
        }
        fileSystem.close();
    }


    //判断某个路径下文件，文件夹
    public static void testListStatus() throws IOException, InterruptedException, URISyntaxException {
        Configuration configuration = new Configuration();
        FileSystem fs = FileSystem.get(new URI("hdfs://niit:9000"), configuration, "atguigu");
        FileStatus[] listStatus = fs.listStatus(new Path("/"));
        for (FileStatus fileStatus : listStatus) {
            if (fileStatus.isFile()) {
                System.out.println("file:"+fileStatus.getPath().getName());
            }else {
                System.out.println("directory:"+ fileStatus.getPath().getName());
            }
        }
    }



}