版本是2.7.4,直接亮代码!
package day_01;
import java.io.ByteArrayOutputStream;
import java.io.DataOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.net.URLConnection;
import java.util.Scanner;import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FsUrlStreamHandlerFactory;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.junit.Test;import com.google.common.io.ByteArrayDataOutput;
import com.microsoft.azure.storage.file.FileOutputStream;public class test {
//使用java原始操作hadoop中的hdfs文件
// @Test
public void readfile() throws IOException{
//注册url流处理器工场(hdfs)
URL.setURLStreamHandlerFactory(new FsUrlStreamHandlerFactory() ) ;
//写入我们要访问hdfs文件系统的文件路劲
URL url =new URL("hdfs://192.168.14.124:8020/user/hadoop1/8.txt");
URLConnection conn = url.openConnection();//连接到文件路径
InputStream is = conn.getInputStream();//使用字节流开读
byte[] by =new byte[is.available()]; //定义字节数组,数组大小为文件大小
// int len=0;
// while((len=is.read())!=-1){ 这是第二种利用循环读,因为我们的数据量不大,就直接一致性读完
// is.read(by, 0, len);
//
// System.out.println(new String(by,0,len));
//
// }
// is.close()
is.read(by);
is.close();
String str = new String(by);//因为我们返回的字节我们可以使用string的构造参数来转为string类型,方便查看内容
System.out.println(str);
}
//使用hadoop api读取文件内容
@Test
public void hdfsreadfile() throws Exception {
Configuration conf=new Configuration();
//当我们使用hadoop api里的东西当然是要指定core.site.xml里配置的,不然他可是不认识的
conf.set("fs.defaultFS", "hdfs://testdata1:8020");
FileSystem fs=FileSystem.get(conf);
Path pa=new Path("/user/hadoop1/8.txt");
FSDataInputStream fis = fs.open(pa);
byte[] by=new byte[1024];
int len=0;
// ByteArrayOutputStream baos=new ByteArrayOutputStream();
java.io.FileOutputStream fos=new java.io.FileOutputStream("E:\\a.txt");
while((len=fis.read(by))!=-1) {
fos.write(by, 0, len);
// System.out.println(new String(by,0,len));
}
fis.close();
fos.close();
// System.out.println(new String(baos.toByteArray()));
}
//使用hadoop api带的ioutils类简化读文件
@Test
public void filereads() throws IOException {
Configuration conf=new Configuration();
conf.set("fs.defaultFS", "hdfs://testdata1:8020");
FileSystem fs=FileSystem.get(conf);
Path pa =new Path("/user/hadoop1/8.txt");
FSDataInputStream read = fs.open(pa);
read.seek(10);
//这个工具底层已经把输出的字节量和流等一些判断都帮我们做了,可以说很强大了
IOUtils.copy(read, System.out);
}
//创建文件夹mkdirs
@Test
public void filemkdir() throws Exception {
Configuration conf=new Configuration();
conf.set("fs.defaultFS","hdfs://testdata1:8020");
FileSystem fs = FileSystem.get(conf);
// FileSystem.mkdirs(fs, new Path("/user/hadoop1/zxz"),这样感觉有点靠谱,我们进行一下简单判断
// new FsPermission(FsAction.ALL, //user
// FsAction.ALL, //group
// FsAction.READ));//other
Path pa =new Path("/user/hadoop1/zxzapp");
if(fs.exists(pa)) {
System.out.println("sorry this path exists");
}else {
FileSystem.mkdirs(fs, pa, new FsPermission(FsAction.ALL,
FsAction.ALL,
FsAction.READ));
System.out.println("success");
}
}
//创建文件create
@Test
public void createfile() throws IOException {
Configuration conf=new Configuration();
conf.set("fs.defaultFS","hdfs://testdata1:8020");
FileSystem fs=FileSystem.get(conf);
Path pa=new Path("/user/hadoop1/zxz1/b.txt");
if(fs.exists(pa)) {
System.out.println("sorry this path exists");
}else {
FSDataOutputStream create = fs.create(fs, pa, new FsPermission(FsAction.ALL,
FsAction.ALL,
FsAction.READ));
create.write("hello".getBytes());
System.out.println("success");
create.close();
}
}
//上传文件copyFromLocalFile
@Test
public void putfile() throws IOException {
Configuration conf=new Configuration();
conf.set("fs.defaultFS","hdfs://testdata1:8020");
FileSystem fs=FileSystem.get(conf);
Path local=new Path("E:\\HBuilder");
Path hdfs=new Path("/user/hadoop1/zxz1/HB");
File file=new File("E:\\HBuilder");
if (!fs.exists(hdfs)&&file.isDirectory()) {
fs.copyFromLocalFile(local, hdfs);
System.out.println("success");
}else {
System.out.println("sorry this path exists");
}
fs.close();
}
//删除文件delete
@Test
public void remove() throws IOException {
Configuration conf=new Configuration();
conf.set("fs.defaultFS","hdfs://testdata1:8020");
FileSystem fs=FileSystem.get(conf);
Scanner sc=new Scanner(System.in);
System.out.println("请输入要删除的文件路径:");
String str = sc.next();
Path pa=new Path(str);
if(fs.exists(pa)) {
fs.delete(pa, true);//true代表递归删除
System.out.println("删除成功");
}else {
System.out.println("删除失败");
}
}
//查看hdfs文件目录的结构
@Test
public void findallfile() throws IOException {
Configuration conf=new Configuration();
conf.set("fs.defaultFS","hdfs://testdata1:8020");
FileSystem fs=FileSystem.get(conf);
Scanner sc=new Scanner(System.in);
System.out.println("请输入要查看的文件路径:");
String str = sc.next();
Path pa=new Path(str);
if(fs.exists(pa)) {
FileStatus[] listStatus = fs.listStatus(pa);
for (FileStatus fileStatus : listStatus) {
Path path = fileStatus.getPath();
System.out.println(path);//显示文件路径
System.out.println(fileStatus.getPermission());//显示文件权限
}
}else {
System.out.println("输入文件有误");
}
}
}
可能格式有点丑,多多包涵!