JAVA访问HDFS(满满干货!基础操作入门)

下面的入门源码里面都说明了功能,不多说(跑之前别忘了开启Hadoop)

import java.net.URI;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;

//读取指定文件,将文件内容输出到控制台
public class ReadData {

	public static void main(String[] args) {
		// TODO Auto-generated method stub
        //定义文件系统路径
		String hdfs_address="hdfs://master:9000";
		//访问的具体文件路径
		String uri=hdfs_address+"/in/words";
		//填写配置信息(默认加载core-default.xml和core-site.xml)
		Configuration conf=new Configuration();
		FSDataInputStream in=null;//输入流字节
		try {
			//第一个参数必须以对象形式传递,不能是字符串
			//第二个参数是配置信息,即上面的Configuration对象
			FileSystem fs=FileSystem.get(URI.create(hdfs_address), conf, "czh");
			in=fs.open(new Path(uri));
		    /*第二个参数是输出到控制台,第三个是缓冲区,
		           第四个是copy完了之后要不要关闭源数据和目的数据
		     */
			//把读到的数据输出到控制台
			IOUtils.copyBytes(in, System.out,4096,false);
		}catch(Exception e) {
			e.printStackTrace();
		}
		finally {
			IOUtils.closeStream(in);//关闭资源
		}
	}

}

import java.net.URI;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

//创建目录
public class CreateDir {

	public static void main(String[] args) {
		// TODO Auto-generated method stub
		String hdfs_address="hdfs://master:9000";
		Configuration conf=new Configuration();
		try {
			FileSystem fs=FileSystem.get(URI.create(hdfs_address), conf, "czh");
			//mkdirs函数会返回一个布尔值来代表是否创建成功
			boolean flag=fs.mkdirs(new Path(hdfs_address+"/my/tmp"));
			System.out.println("Directory create"+(flag?"successfully":"failed"));
		}catch(Exception e) {
			e.printStackTrace();
		}

	}

}

import java.net.URI;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

//上传文件
public class PutFile {

	public static void main(String[] args) {
		// TODO Auto-generated method stub
		String hdfs_address="hdfs://master:9000";
		try {
			FileSystem fs=FileSystem.get(URI.create(hdfs_address), new Configuration(), "czh");
		    Path src=new Path("D:\\a.txt");//注意分割符与Windows下相反,此文件必须存在!
		    Path dest=new Path(hdfs_address+"/my");//上传目标路径
		    fs.copyFromLocalFile(src, dest);
		}catch(Exception e) {
			e.printStackTrace();
		}

	}

}

import java.io.FileOutputStream;
import java.net.URI;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;

//下载文件,并写入到本地D:\\my\\路径下的b.txt文件中
public class DownFile {

	public static void main(String[] args) {
		// TODO Auto-generated method stub
		String hdfs_address="hdfs://master:9000";
		try {
			FileSystem fs=FileSystem.get(URI.create(hdfs_address), new Configuration(), "czh");
		    Path src=new Path(hdfs_address+"/my/a.txt");
		    FileOutputStream out=new FileOutputStream("D:\\my\\b.txt");
		    //open函数用于创建一个输入流,起点是src
		    FSDataInputStream in=fs.open(src);
		    IOUtils.copyBytes(in, out, 4096,false);
;		}catch(Exception e) {
			e.printStackTrace();
		}

	}

}

import java.net.URI;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

//列出目录下的文件
public class ListFile {

	public static void main(String[] args) {
		// TODO Auto-generated method stub
		String hdfs_address="hdfs://master:9000";
		try {
			FileSystem fs=FileSystem.get(URI.create(hdfs_address), new Configuration(), "czh");
		    FileStatus files[]=fs.listStatus(new Path(hdfs_address+"/my"));
		    for(FileStatus file:files) {
		    	System.out.println(file.getPath());
		    }
;		}catch(Exception e) {
			e.printStackTrace();
		}

	}

}

import java.net.URI;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

//删除文件
public class DeleteFile {

	public static void main(String[] args) {
		// TODO Auto-generated method stub
		String hdfs_address="hdfs://master:9000";
		try {
			FileSystem fs=FileSystem.get(URI.create(hdfs_address), new Configuration(), "czh");
		    boolean flag=fs.delete(new Path(hdfs_address+"/my"),true);//第二个参数是是否递归删除
		    System.out.println("File deleted:"+flag);
;		}catch(Exception e) {
			e.printStackTrace();
		}

	}

}

猜你喜欢

转载自blog.csdn.net/weixin_44593822/article/details/106485664