搭建 hadoop 3.1.2 独立模式,单节点和多节点伪分布式安装与使用
在 AngularJS + SpringMVC 多文件 本地 上传和下载 上增加hdfs的上传和下载
添加依赖
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>3.1.2</version>
</dependency>
application.yml 新增hdfs地址和所需上传的文件夹
hdfs:
serverUrl: hdfs://hadoop-master:9000
uploadDir: /upload/
hdfs上传下载工具类
@Component
public class HdfsUtil {
private static final Logger logger = LoggerFactory.getLogger(HdfsUtil.class);
private static String hdfsServerUrl;
private static String hdfsUploadDir;
@Value("${hdfs.serverUrl:''}")
public void setHdfsServerUrl(String serverUrl) {
hdfsServerUrl = serverUrl;
}
@Value("${hdfs.uploadDir:''}")
public void setHdfsUploadDir(String uploadDir) {
hdfsUploadDir = uploadDir;
}
static Configuration conf;
static FileSystem fs;
static{
try {
conf = new Configuration();
fs = FileSystem.newInstance(conf);
} catch (IOException e) {
logger.error("初始化 FileSystem 发生异常:{}", e.getMessage());
}
Runtime.getRuntime().addShutdownHook(new Thread(){
@Override
public void run(){
try{
fs.close();
}catch (IOException e) {
logger.error("关闭 FileSystem 发生异常:{}", e.getMessage());
}
}
});
}
/**
* 上传文件
* @param file
* @return
*/
public static String uploadFile(MultipartFile file){
String fileName = file.getOriginalFilename();
String filePath = hdfsUploadDir + fileName;
try {
InputStream inputStream = file.getInputStream();
Path path = new Path(filePath);
OutputStream outputStream = fs.create(path);
IOUtils.copyBytes(inputStream,outputStream,4096,true);
}catch (IOException e){
logger.error("文件上传发生异常:{}", e.getMessage());
}
return hdfsServerUrl + filePath;
}
/**
* 下载文件
* @param filePath
* @param response
*/
public static void downloadFile(String filePath, HttpServletResponse response){
String fileName = filePath.substring(filePath.lastIndexOf('/') + 1);
String hdfsFilePath = filePath.replace(hdfsServerUrl,"");
response.setCharacterEncoding ("utf-8");
//二进制流
response.setContentType("application/octet-stream");
try(
InputStream inputStream = fs.open(new Path(hdfsFilePath));
ServletOutputStream servletOutputStream = response.getOutputStream();
){
response.setHeader("Content-Disposition", "attachment;filename=" + new String(fileName.getBytes("GB2312"), "ISO8859-1"));
int len;
byte[] buffer = new byte[1024];
while ((len = inputStream.read(buffer)) > 0){
servletOutputStream.write(buffer,0, len);
}
}catch (IOException e){
logger.error ("文件下载发生异常:{}", e.getMessage());
}
}
}
1. 上传
新增hdfs上传按钮
<button class="btn" type="submit" ng-click="fileUploadHdfs()">上传Hdfs</button>
上传hdfs点击事件
//上传hdfs
$scope.fileUploadHdfs = function () {
var form = new FormData();
var selectFiles = $scope.repeatFiles;
for (var i=0; i<selectFiles.length; i++) {
form.append("file", selectFiles[i]);
}
var url = 'file/uploadHdfs';
upload(url,form);
};
function upload(url,form) {
$http({
method: 'POST',
url: Config.getApiRemoteUrl() + url,
data: form,
headers: {
'Content-Type':undefined
},
transformRequest: angular.identity
}).then(function successCallback(response) {
if(response.status === 200){
layer.msg("success");
}else{
layer.msg("fail");
}
}, function errorCallback(response) {
layer.msg("error");
});
}
后端上传接口,调用hdfs工具类上传
@PostMapping("/uploadHdfs")
public void uploadHdfs(@RequestParam(value = "file") MultipartFile[] files){
if(files != null && files.length > 0){
for(MultipartFile file : files){
HdfsUtil.uploadFile(file);
}
}
}
效果
2. 下载
新增下载hdfs的a标签
<a target="_self" id="downloadHdfs" ng-click="fileDownloadHdfs()">下载Hdfs</a>
替换a标签的超链接url,并没有把hdfs文件路径当参数放进去
//下载hdfs
$scope.fileDownloadHdfs = function () {
$('#downloadHdfs').attr('href', './api/file/downloadHdfs');
};
后端下载接口,调用hdfs工具类下载
@GetMapping("/downloadHdfs")
public void downloadHdfs(HttpServletResponse response) {
String filePath = "hdfs://hadoop-master:9000/upload/upload1.txt";
HdfsUtil.downloadFile(filePath,response);
}
效果