execl上传功能,是一个经常遇到的功能,无非包括解析execl,把解析的数据存储到数据库。我最近一个项目也用到了execl上传,由于execl数据量比较小,是通过同步实现了execl解析,然后把解析的数据存到数据库,有个明显的缺点,当execl数据在7000条记录左右,耗时大概是10min。这个速度有点慢了,一般解决方法有两个,一个是利用批作业,凌晨服务器空闲时跑批作业实现;还有一个方法就是实现异步上传;这几天研究了一下异步上传execl,把代码实现了下,供大家参考学习。
异步上传实现思路:利用AOP实现execl上传并记录上传失败的数据。本示例不但包括AOP相关内容,还用到自定义注解。还有一个重点,就是前端一个execl上传附件控件,但是可以实现上传不同的execl,也就是不同execl对应不同的bo类型,我是通过一个方法实现的,就是readExcel方法,大家可以看看。
废话不说了,上代码......
controller部分
package com.lsl.mylsl.controller;
import com.lsl.mylsl.service.IUploadExeclSyncService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.multipart.MultipartFile;
import java.math.BigDecimal;
@RestController
@RequestMapping("/api/lsl")
public class UploadExeclSyncController {
@Autowired
IUploadExeclSyncService uploadExeclSyncService;
/**
* 前端一个附件上传功能,可以上传不同execl,利用boType标识execl对应的bo
* @param mulFile execl附件
* @param boType 标识execl类型
* @return
*/
@PostMapping("upload")
public String uploadExeclToDb(@RequestParam(value = "file",required = false) MultipartFile mulFile, @RequestParam(value = "boType") String boType){
long sizeB = mulFile.getSize();
float sizeM = Float.parseFloat(String.valueOf(sizeB)) / 1024 / 1024;
BigDecimal bg = new BigDecimal(String.valueOf(sizeM));
sizeM = bg.setScale(2,BigDecimal.ROUND_HALF_UP).floatValue();
if (sizeM > 30){
return "execl附件不允许大于30M";
}
//这里result返回的是切面里的返回值(成功)
String result = uploadExeclSyncService.excelToDb(mulFile,boType);
return result;
}
}
service部分
package com.lsl.mylsl.service;
import com.lsl.mylsl.BO.BaseBO;
import org.springframework.web.multipart.MultipartFile;
import java.util.List;
public interface IUploadExeclSyncService {
List<BaseBO> readExcel(MultipartFile mulFile,String boType) throws Exception;
String excelToDb(MultipartFile mulFile, String boType);
}
excelToDb这个方法加了自定义注解@UploadExecl
package com.lsl.mylsl.service.impl;
import com.lsl.mylsl.BO.BaseBO;
import com.lsl.mylsl.BO.CatBO;
import com.lsl.mylsl.BO.MokeyBO;
import com.lsl.mylsl.annotation.UploadExecl;
import com.lsl.mylsl.service.IUploadExeclSyncService;
import org.apache.poi.hssf.usermodel.HSSFWorkbook;
import org.apache.poi.ss.usermodel.*;
import org.springframework.stereotype.Service;
import org.springframework.web.multipart.MultipartFile;
import java.io.InputStream;
import java.text.DecimalFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.List;
@Service
public class UploadExeclSyncServiceImpl implements IUploadExeclSyncService {
/**
* 一个通用的读取execl数据的方法
* 例如:有多个execl,每个execl的字段不同,但是要通过一个附件上传并读取execl的内容存储到数据库
* 前端有一个下拉列表框来表示那种类型execl,既boType;就是看execl的数据应该映射那个bo
* @param mulFile
* @param boType
* @return
*/
@Override
public List<BaseBO> readExcel(MultipartFile mulFile, String boType) throws Exception{
String filename = mulFile.getOriginalFilename();
InputStream inputStream = mulFile.getInputStream();
boolean isXls = isXls(filename);
Workbook workbook = null;
if (isXls){
workbook = new HSSFWorkbook(inputStream);
}else {
throw new Exception("文件格式不对,请选择(xls,xlsx,et)文件上传");
}
//获取第一个sheet
Sheet sheet = workbook.getSheetAt(0);
//获取第一行,一般是标题行
Row titleRow = sheet.getRow(0);
//获取列数
int lastCellNum = titleRow.getLastCellNum();
//获取行数
int lastRowNum = sheet.getLastRowNum();
List<BaseBO> baseBOList = new ArrayList<>();
for (int i=1;i<=lastRowNum;i++){
Row row = sheet.getRow(i);
//判断是否为空行,是空行跳过,不要写库
boolean isEmpty = isRowEmpty(row);
if (isEmpty){
continue;
}
//这里不同execl对应的bo都继承了basebo
BaseBO baseBO = getSubClassType(boType);
//遍历列
for (int j=0;j<lastCellNum;j++){
Cell cell = row.getCell(j);
baseBO = getExeclCell(baseBO,j,boType,cell);
}
baseBOList.add(baseBO);
}
workbook.close();
return baseBOList;
}
/**
* 把execl数据导入数据库
* @param mulFile
* @param boType
* @return
*/
@UploadExecl
@Override
public String excelToDb(MultipartFile mulFile, String boType) {
try {
//从execl读取数据到list 这里可以把readExecl改造成读取execl一行记录
List<BaseBO> execlList = readExcel(mulFile,boType);
for (BaseBO bo : execlList){
//把execl数据存到数据库
saveToDb(bo);
}
} catch (Exception e) {
return "fail";
}
return "success";
}
/**
* 判断文件是否为xls,xlsx,et格式
* @param fileName
* @return
* @throws Exception
*/
public boolean isXls(String fileName) throws Exception{
if (fileName.matches("^.+\\.(?i)(xls)$")){
return true;
}else if (fileName.matches("^.+\\.(?i)(xlsx)$")){
return true;
}else if (fileName.matches("^.+\\.(?i)(et)$")){
return true;
}else {
return false;
}
}
/**
* 判断是否是空行
* @param row
* @return
*/
public boolean isRowEmpty(Row row){
for (int n = row.getFirstCellNum();n<row.getLastCellNum();n++){
Cell cell = row.getCell(n);
if (cell != null && cell.getCellType() != CellType.BLANK){
return false;
}
}
return true;
}
/**
*
* @param boType
* @return
*/
public BaseBO getSubClassType(String boType){
BaseBO baseBo = null;
if ("cat".equals(boType)){
baseBo = new CatBO();
}else if ("mokey".equals(boType)){
baseBo = new MokeyBO();
}
return baseBo;
}
/**
* 读取execl单元格的数据
* @param baseBO
* @param j
* @param boType
* @param cell
* @return
*/
public BaseBO getExeclCell(BaseBO baseBO,int j,String boType,Cell cell){
if ("cat".equals(boType)){
return cellToCatBo(baseBO,j,cell);
}else if ("mokey".equals(boType)){
//这里逻辑和if分支类似
return new BaseBO();
}else {
return new BaseBO();
}
}
/**
* 获取execl记录映射到对应的子类bo中
* @param baseBO
* @param j
* @param cell
* @return
*/
public BaseBO cellToCatBo(BaseBO baseBO,int j,Cell cell){
CatBO catBO = (CatBO)baseBO;
if (j==0){//读取第一个单元格内容
catBO.setCatAge(getCellValue(cell));
}
if (j==1){//读取第二个单元格内容
catBO.setCatName(getCellValue(cell));
}
return catBO;
}
/**
* 获取单元格内容
* @param cell
* @return
*/
public String getCellValue(Cell cell){
String cellValue = "";
if (null == cell){
return cellValue;
}
CellType cellType = cell.getCellType();
switch (cellType){
case STRING:
cellValue = cell.getStringCellValue();
break;
case NUMERIC:
if ("General".equals(cell.getCellStyle().getDataFormatString())){
DecimalFormat df = new DecimalFormat("0");
cellValue = df.format(cell.getNumericCellValue());
}else if ("yyyy\\-mm\\-dd\\ hh:mm:ss".equals(cell.getCellStyle().getDataFormatString())){
SimpleDateFormat sdf1 = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
cellValue = sdf1.format(cell.getDateCellValue());
}else if ("yyyy\\-mm\\-dd\\".equals(cell.getCellStyle().getDataFormatString())){
SimpleDateFormat sdf2 = new SimpleDateFormat("yyyy-MM-dd");
cellValue = sdf2.format(cell.getDateCellValue());
}else if ("m/d/yy".equals(cell.getCellStyle().getDataFormatString())){
SimpleDateFormat sdf3 = new SimpleDateFormat("yyyy/MM/dd");
cellValue = sdf3.format(cell.getDateCellValue());
}else if ("m/d/yy h:mm".equals(cell.getCellStyle().getDataFormatString())){
SimpleDateFormat sdf4 = new SimpleDateFormat("yyyy/MM/dd HH:mm");
cellValue = sdf4.format(cell.getDateCellValue());
}else if ("yyyy/mm/dd\\ hh:mm:ss".equals(cell.getCellStyle().getDataFormatString())){
SimpleDateFormat sdf5 = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss");
cellValue = sdf5.format(cell.getDateCellValue());
}else {
}
break;
case BLANK:
break;
default:
}
return cellValue;
}
/**
* 把execl数据存到数据库
* @param baseBO
* @return
*/
public boolean saveToDb(BaseBO baseBO){
//把数据存储到数据库
return true;
}
}
BO部分
package com.lsl.mylsl.BO;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableField;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import java.io.Serializable;
@TableName("tab_cat")
public class CatBO extends BaseBO implements Serializable {
private static final long serialVersionUID = 1765207774189824729L;
@TableId(type = IdType.UUID)
private String catId;
@TableField("CAT_NAME")
private String catName;
@TableField("CAT_AGE")
private String catAge;
//该字段在数据库中不存在,业务需要,不会映射到表
@TableField(exist = false)
private String catNum;
public String getCatId() {
return catId;
}
public void setCatId(String catId) {
this.catId = catId;
}
public String getCatName() {
return catName;
}
public void setCatName(String catName) {
this.catName = catName;
}
public String getCatAge() {
return catAge;
}
public void setCatAge(String catAge) {
this.catAge = catAge;
}
public String getCatNum() {
return catNum;
}
public void setCatNum(String catNum) {
this.catNum = catNum;
}
@Override
public String toString() {
return "CatBO{" +
"catId='" + catId + '\'' +
", catName='" + catName + '\'' +
", catAge='" + catAge + '\'' +
", catNum='" + catNum + '\'' +
'}';
}
}
注意这个BaseBO,这个事readExecl方法实现可以解析不同execl的关键
package com.lsl.mylsl.BO;
public class BaseBO {
}
切面部分,核心
package com.lsl.mylsl.Utils;
import org.aspectj.lang.ProceedingJoinPoint;
import org.aspectj.lang.annotation.Around;
import org.aspectj.lang.annotation.Aspect;
import org.aspectj.lang.annotation.Pointcut;
import org.springframework.stereotype.Component;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@Component
@Aspect
public class UploadExeclAspect {
ExecutorService executor = Executors.newFixedThreadPool(5);
@Pointcut("@annotation(com.lsl.mylsl.annotation.UploadExecl)")
public void uploadPoint() {}
@Around(value = "uploadPoint()")
public Object uploadControl(ProceedingJoinPoint pjp) {
//获取目标方法名
String methodName = pjp.getSignature().getName();
//获取目标方法的入参
Object[] params = pjp.getArgs();
List<Object> objects = Arrays.asList(params);
System.err.println("UploadExeclAspect--params = " + objects);
executor.submit(()->{
try {
//执行目标方法,这里返回的是目标方法excelToDb返回的值(success/fail)
String result = (String) pjp.proceed();
if ("fail".equals(result)){
//把失败数据存储记录下
saveFailData(objects);
}
} catch (Throwable throwable) {
throwable.printStackTrace();
}
});
return "成功";
}
/**
* 把存储失败的数据记录下
*/
public void saveFailData(List<Object> objects){
System.out.println("数据存储失败,数据信息为 = " + objects);
}
}
注解@UploadExecl部分
package com.lsl.mylsl.annotation;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
@Retention(RetentionPolicy.RUNTIME)
@Target({ElementType.METHOD})
public @interface UploadExecl {
}