redis 使用经验总结
①、定时更新天气网数据文件解析并存储到redis
②、通过接口访问形式提供给高德
1、定时更新天气网数据文件解析并存储到redis
- 下载GridForecast、GridHourly、GridNow、RainMinute四类数据文件 同时在服务器本地备份
date = TqwUtil.formatDate(TqwUtil.parse(date, dateFormat), dateFormat); //设置当前时间 RedisUtil.set("{" + TqwConstant.GRID_NOW_TIME + "}", date);
map.put("{" + TqwConstant.RedisKeyPrefix.GRID_NOW + lonlatString + "}", sb.toString()); //todo 以后删除,3*3 造出 1*1 的数据, 用中心点填充周围 BigDecimal lonBigdecimal = new BigDecimal(lon); BigDecimal latBigdecimal = new BigDecimal(lat); BigDecimal lonIncr = lonBigdecimal.add(new BigDecimal("0.01")); BigDecimal latIncr = latBigdecimal.add(new BigDecimal("0.01")); BigDecimal lonDecr = lonBigdecimal.subtract(new BigDecimal("0.01")); BigDecimal latDecr = latBigdecimal.subtract(new BigDecimal("0.01")); map.put("{" + TqwConstant.RedisKeyPrefix.GRID_NOW + TqwUtil.formatLonLatString(lon, latIncr.toString()) + "}", sb.toString()); map.put("{" + TqwConstant.RedisKeyPrefix.GRID_NOW + TqwUtil.formatLonLatString(lon, latDecr.toString()) + "}", sb.toString()); map.put("{" + TqwConstant.RedisKeyPrefix.GRID_NOW + TqwUtil.formatLonLatString(lonIncr.toString(), lat) + "}", sb.toString()); map.put("{" + TqwConstant.RedisKeyPrefix.GRID_NOW + TqwUtil.formatLonLatString(lonDecr.toString(), lat) + "}", sb.toString()); map.put("{" + TqwConstant.RedisKeyPrefix.GRID_NOW + TqwUtil.formatLonLatString(lonIncr.toString(), latIncr.toString()) + "}", sb.toString()); map.put("{" + TqwConstant.RedisKeyPrefix.GRID_NOW + TqwUtil.formatLonLatString(lonIncr.toString(), latDecr.toString()) + "}", sb.toString()); map.put("{" + TqwConstant.RedisKeyPrefix.GRID_NOW + TqwUtil.formatLonLatString(lonDecr.toString(), latIncr.toString()) + "}", sb.toString()); map.put("{" + TqwConstant.RedisKeyPrefix.GRID_NOW + TqwUtil.formatLonLatString(lonDecr.toString(), latDecr.toString()) + "}", sb.toString()); if ( map.size() >= DataUtil.object2Int(Constant.PROPERTIES.get("pipeline.count")) ) { RedisUtil.setByPipeline(map); map.clear(); }通过Redis String数据类型存储,每次存储覆盖上次数据,四类数据均采用此数据类型。尝试使用Hash,但是由于Hash内部存储数据大小有限制,存储信息超过一定大小,内存溢出,不能继续存储,虽然机器内容仍然够用。
- 将备份文件存储在缓存中,方便备份请求获取数据 空间换时间的概念
public File getGridNowFile(String[] arr) { logger.info(" grid now start back up !"); String filename = TqwConstant.BASE_PATH + File.separator + RegexUtil.getRandomString(10) + ".txt"; //打包文件 数组 try { String dataFormat = "yyyyMMddHHmm"; BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(filename))); String gridNowStart = RedisUtil.get("{" + TqwConstant.GRID_NOW_TIME + "}"); DateTime dateTime = new DateTime(TqwUtil.parse(gridNowStart, dataFormat)); bw.write(dateTime.toString("yyyyMMddHH") + "00"); bw.newLine(); List<String> keys = new ArrayList<>(); for ( int i = 0; i < arr.length; i++ ) { String[] lonlat = arr[i].split(","); String lonlatStr = TqwUtil.formatLonLatString(lonlat[0], lonlat[1]); keys.add("{" + TqwConstant.RedisKeyPrefix.GRID_NOW + lonlatStr + "}"); if ( keys.size() >= 10000 ) { List<String> values = RedisUtil.getByPipeline(keys, String.class); for ( int j = 0; j < values.size(); j++ ) { String value = values.get(j); if ( TqwUtil.isEmpty(value) ) { bw.write("######################################"); } else { bw.write(value); } bw.newLine(); } keys.clear(); } } if ( keys.size() > 0 ) { List<String> values = RedisUtil.getByPipeline(keys, String.class); for ( int j = 0; j < values.size(); j++ ) { String value = values.get(j); if ( TqwUtil.isEmpty(value) ) { bw.write("######################################"); } else { bw.write(value); } bw.newLine(); } } IOUtils.closeQuietly(bw); } catch ( Exception e ) { logger.error(" 生成格点实况数据异常 ", e); FileUtils.deleteQuietly(new File(filename)); return null; } logger.info(" grid now end back up "); String zipFilename = TqwConstant.BASE_PATH + File.separator + RegexUtil.getRandomString(10) + ".zip"; try { ZipUtil.zip(zipFilename, "", filename); logger.info(" grid now end package "); } catch ( Exception e ) { logger.error("Grid Now zip error!", e); } finally { FileUtils.deleteQuietly(new File(filename)); } return new File(zipFilename); }
File zipFile = this.backFileGenerator.getGridNowFile(array); try { ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); FileUtils.copyFile(zipFile, byteArrayOutputStream); TqwRedisUtil.set("{" + TqwConstant.FileBytesKey.NOW_BYTES + "}", byteArrayOutputStream.toByteArray()); IOUtils.closeQuietly(byteArrayOutputStream);
2.高德请求数据
根据经纬度数组返回经纬度对应的数据
public File getGridNowFile(String[] arr) { logger.info(" grid now start generate "); String filename = TqwConstant.BASE_PATH + "/grid-now-" + RegexUtil.getRandomString(10) + ".txt"; File file = new File(filename); try { String dataFormat = "yyyyMMddHHmm"; BufferedWriter br = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(file))); String gridNowStart = RedisUtil.get("{" + TqwConstant.GRID_NOW_TIME + "}"); DateTime dateTime = new DateTime(TqwUtil.parse(gridNowStart, dataFormat)); br.write(dateTime.toString("yyyyMMddHH") + "00"); br.newLine(); List<String> requestKeys = new ArrayList<>(); List<String> keys = new ArrayList<>(); for ( int i = 0; i < arr.length; i++ ) { String[] lonlat = arr[i].split(","); boolean isValid = TqwUtil.isValid(lonlat[0], lonlat[1]); if ( !isValid ) { continue; } double[] lonLatArray = CoordinateUtil.gcj02ToBd09(Double.valueOf(lonlat[0]), Double.valueOf(lonlat[1])); String lonlatStr = TqwUtil.formatLonLatString(String.valueOf(lonLatArray[0]), String.valueOf(lonLatArray[1])); keys.add("{" + TqwConstant.RedisKeyPrefix.GRID_NOW + lonlatStr + "}"); requestKeys.add(arr[i] + "_"); if ( requestKeys.size() >= 10000 ) { List<String> values = RedisUtil.getByPipeline(keys, String.class); for ( int j = 0; j < values.size(); j++ ) { br.write(requestKeys.get(j)); String value = values.get(j); if ( TqwUtil.isEmpty(value) ) { br.write("#########################"); } else { br.write(value.substring(0, 25)); } br.newLine(); } keys.clear(); requestKeys.clear(); } } if ( requestKeys.size() > 0 ) { List<String> values = RedisUtil.getByPipeline(keys, String.class); for ( int j = 0; j < values.size(); j++ ) { br.write(requestKeys.get(j)); String value = values.get(j); if ( TqwUtil.isEmpty(value) ) { br.write("#########################"); } else { br.write(value.substring(0, 25)); } br.newLine(); } } IOUtils.closeQuietly(br); } catch ( Exception e ) { logger.error(" 生成格点实况数据异常 ", e); FileUtils.deleteQuietly(file); return null; } logger.info(" grid now end generate "); String zipFilename = TqwConstant.BASE_PATH + "/grid-now-" + RegexUtil.getRandomString(10) + ".zip"; try { ZipUtil.zip(zipFilename, "", filename); logger.info(" grid now end package "); } catch ( Exception e ) { logger.error("Grid Now zip error!", e); } finally { FileUtils.deleteQuietly(file); } return new File(zipFilename); }
总结: