工程所依赖的jar包, hadoop-1.1.2/*.jar、hadoop-1.1.2/lib/*.jar
编写Java程序通过URL访问HDFS
public class HDFSTest { public static final String HDFSPATH ="hdfs://centos:9000/hello.txt"; @Test public void testAcceHDFSForURL() throws IOException{ URL url = new URL(HDFSPATH); InputStream inputStream = url.openStream(); IOUtils.copyBytes(inputStream, System.out, 1024, true); } } java.net.MalformedURLException: unknown protocol: hdfs
只是这样访问,程序会有异常,畸形的URL,就是不能解析HDFS这种协议,使用setURLStreamHandlerFactory()让URL能够识别hdfs协议
public class HDFSTest { //URL如果不是用IP地址,使用主机名,需要在hosts文件中配置主机名到IPd地址的映射 public static final String HDFSPATH ="hdfs://192.168.56.101:9000/hello.txt"; @Test public void testAcceHDFSForURL() throws IOException{ URL.setURLStreamHandlerFactory(new FsUrlStreamHandlerFactory()); URL url = new URL(HDFSPATH); InputStream inputStream = url.openStream(); /** InputStream:输入流 OutputStream:输出流 int:缓冲区大小 boolean:传输结束后是否关闭流 **/ IOUtils.copyBytes(inputStream, System.out, 1024, true); } }
使用HDFS提供的接口访问HDFS,功能更加强大,操作类FileSystem
删除HDFS中的所有文件、目录
[root@centos Downloads]# hadoop fs -rmr hdfs://centos:9000/* Warning: $HADOOP_HOME is deprecated. Deleted hdfs://centos:9000/file1 Deleted hdfs://centos:9000/hello.txt Deleted hdfs://centos:9000/usr [root@centos Downloads]# hadoop fs -lsr / Warning: $HADOOP_HOME is deprecated. [root@centos Downloads]# hadoop fs -lsr / Warning: $HADOOP_HOME is deprecated.创建一个名为file1的目录
public static final String HDFSURIPATH ="hdfs://192.168.56.101:9000/"; @Test public void testFileSystemForUploadFile() throws IOException, URISyntaxException{ FileSystem fileSystem = getFileSystem(); fileSystem.mkdirs(new Path("/file1")); } private FileSystem getFileSystem() throws IOException, URISyntaxException { //Configuration对象加载配置文件 FileSystem fileSystem = FileSystem.get(new URI(HDFSURIPATH), new Configuration()); return fileSystem; }
通过命令查看文件创建成功
[root@centos Downloads]# hadoop fs -lsr / Warning: $HADOOP_HOME is deprecated. drwxr-xr-x - Administrator supergroup 0 2015-05-22 03:49 /file1
删除目录
@Test public void testFileSystemForDelete() throws IOException, URISyntaxException{ FileSystem fileSystem = getFileSystem(); fileSystem.delete(new Path("/file1"),true); }
命令查看
[root@centos Downloads]# hadoop fs -lsr / Warning: $HADOOP_HOME is deprecated.
上传文件
@Test public void testFileSystemForCreate() throws IOException, URISyntaxException{ //创建一个名称为file1的目录 testFileSystemForMkdir(); //创建一个文件 FSDataOutputStream fsOutputStream = getFileSystem().create(new Path("/file1/hello.txt")); //获取本地文件 FileInputStream fInputStrea = new FileInputStream("E:/Linux/Linux.txt"); //把本地文件E:/Linux/Linux.txt中的内容拷贝到hdfs中/file/hello.txt文件中 IOUtils.copyBytes(fInputStrea, fsOutputStream, 1024, true); }
另一种上传文件的方式
FileSystem fs = HDFSUtils.getFileSystem(); Path s = new Path("E:/Linux/201207141451233056.png"); Path d = new Path("/file1/"); fs.copyFromLocalFile(s, d);
命令查看
[root@centos Downloads]# hadoop fs -lsr / Warning: $HADOOP_HOME is deprecated. drwxr-xr-x - Administrator supergroup 0 2015-05-22 04:14 /file1 -rw-r--r-- 3 Administrator supergroup 4617 2015-05-22 04:14 /file1/hello.txt
读取文件,中文内容会有乱码
@Test public void testFileSystemForOpen() throws IOException, URISyntaxException{ FSDataInputStream fsInputStream = getFileSystem().open(new Path("/file1/hello.txt")); IOUtils.copyBytes(fsInputStream, System.out, 1024, true); }
查看目录及文件
@Test public void testFileSystemForListStatus() throws IOException, URISyntaxException{ FileStatus[] listStatus = getFileSystem().listStatus(new Path("/file1")); for (FileStatus fs : listStatus) { System.out.println(fs.getLen()); System.out.println(fs.isDir()); System.out.println(fs.getPath()); System.out.println(fs.getPermission()); System.out.println(fs.getReplication()); } }