需求
获取200多家门店,各个门店上月的相关消费情况的会员数量(近两月消费一次会员数、当月消费一次会员数、消费两次会员数)
问题
会员数量过多(三百七十万),获取某一个门店满足相关消费情况的会员数量的查询sql查询时间过长(远远超过当前项目所设置的单个sql运行时间不得超过1秒的时间设定)
解决方式:
1、使用ThreadPoolTaskScheduler
进行线程任务调度,单次处理一小部分会员(我这边写的是单次处理1万条),并且进行sql优化,确保单个sql运行时间不超过1秒
2、使用Redis String 的increment
将满足条件的会员数量累加。
注意:使用CountDownLatch
对线程进行控制,确保Redis将所有满足条件的会员全部累加完成后再返回数据
public class ServiceReport{
Logger logger = LoggerFactory.getLogger(ServiceReport.class);
@Autowired
ReportMapper reportMapper;
@Autowired
ShopMapper shopMapper;
@Autowired
MemberMapper memberMapper;
@Autowired
RedisTemplate redisTemplate;
public List<DtoMonthReport> reduceShopMonthReport(){
//创建ThreadPoolTaskScheduler 并进行相关设置
ThreadPoolTaskScheduler taskScheduler = new ThreadPoolTaskScheduler();
taskScheduler.setPoolSize(32);
taskScheduler.setThreadNamePrefix("test-");
taskScheduler.setWaitForTasksToCompleteOnShutdown(true);
taskScheduler.setAwaitTerminationSeconds(60);
taskScheduler.initialize();
Date lastMonthFirstTime = HelpApp.getLastMonthFirstTime(); //上月月初
Date lastMonthLastTime = HelpApp.getLastMonthLastTime(); //上月月末
Date lastTwoMonthFirstTime = HelpApp.getLastTwoMonthFirstTime(); //前两月月初
List<String> shopCodes = shopMapper.getAllShopCode();//拉取所有月报门店 250左右
List<DtoMonthReport> dtoMonthReports = new ArrayList<>();
for (String shop: shopCodes) {
try {
int lastMemberId = memberMapper.getLastId(); //最后一个会员id 推测总会员数 3700000左右
DtoMonthReport monthReport = reportMapper.getRegionMonthReport(shop, lastMonthFirstTime);
//会员数量过多总,每次处理10000个会员的消费情况
int page = lastMemberId/10000+1;
int no = 0;
int beginId;
int endId;
//使用CountDownLatch控制线程执行次数
CountDownLatch latch2 = new CountDownLatch(page);
while (page > 0){
page --;
beginId = 1+no*10000;
endId = (no+1)*10000;
int finalBeginId = beginId;
int finalEndId = endId;
//启动我们创建并配置好的ThreadPoolTaskScheduler,达到多线程运行下列代码块的目的
taskScheduler.execute(()->{
try {
int consumeOnceOfTwoMonth = reportMapper.getRegionConsumeOnceOfTwoMonth(shop,lastTwoMonthFirstTime,lastMonthLastTime , finalBeginId, finalEndId); //近两月消费1次会员数
int consumeOnce = reportMapper.getRegionConsumeOnce(shop, lastMonthFirstTime, lastMonthLastTime , finalBeginId, finalEndId); //消费1次会员数
int consumeTwice = reportMapper.getRegionConsumeTwice(shop, lastMonthFirstTime, lastMonthLastTime , finalBeginId, finalEndId); //消费2次会员数
//使用redis String的increment 将每个线程中符合相关消费情况的会员数量叠加
redisTemplate.opsForValue().increment("produceShopReport:"+shop+":consumeOnceOfTwoMonth", consumeOnceOfTwoMonth);
redisTemplate.opsForValue().increment("produceShopReport:"+shop+":consumeOnce", consumeOnce);
redisTemplate.opsForValue().increment("produceShopReport:"+shop+":consumeTwice", consumeTwice);
} catch (Exception e) {
e.printStackTrace();
} finally {
latch2.countDown();
}
});
no++;
}
//使用CountDownLatch 的await() , 确保程序在所有线程查询数据结束之后再往下运行
try {
latch2.await();
} catch (InterruptedException e) {
e.printStackTrace();
}
monthReport.setShop(shop);
//将redis计算好的数据取出
monthReport.setConsumeOnceOfTwoMonth((int)redisTemplate.opsForValue().get("produceShopReport:"+shop+":consumeOnceOfTwoMonth"));
monthReport.setConsumeOnce((int)redisTemplate.opsForValue().get("produceShopReport:"+shop+":consumeOnce"));
monthReport.setConsumeTwice((int)redisTemplate.opsForValue().get("produceShopReport:"+shop+":consumeTwice"));
dtoMonthReports.add(monthReport);
} catch (Exception e) {
logger.error(shop+"--get month report data failed--"+e);
}
}
//此次需求需要的数据拉取完毕
logger.info("produceShopReport ready Data success");
return dtoMonthReports;
}
}