package com.test.yuhua.common.util; import org.apache.commons.lang3.time.DateUtils; import java.util.Calendar; import java.util.Date; public final class DateUtil { private static final Date UNIX_DATE; static { Calendar calendar = Calendar.getInstance(); calendar.set(1970,1,1,0,0,0); calendar.set(Calendar.MILLISECOND,0); UNIX_DATE = calendar.getTime(); } public static long getAfterUnixDays(Date currentDate){ return getBetweenDays(UNIX_DATE,currentDate); } public static long getBetweenDays(Date beginDate,Date endDate){ long num = (endDate.getTime() - beginDate.getTime()) / (24 * 60 * 60 * 1000); return num; } public static Date addDays(Date date, int days) { return DateUtils.addDays(date, days); } public static Date addSeconds(Date date, int secs) { return DateUtils.addSeconds(date, secs); }
}
package com.test.yuhua.service.dynamic.shard; import com.dangdang.ddframe.rdb.sharding.api.ShardingValue; import com.dangdang.ddframe.rdb.sharding.api.strategy.table.SingleKeyTableShardingAlgorithm; import com.google.common.collect.Range; import com.test.yuhua.common.util.DateUtil; import java.util.Collection; import java.util.Date; import java.util.LinkedHashSet; public class ServerTablesShardingDateTypeAlgorithm implements SingleKeyTableShardingAlgorithm<Date> { private int tableNums; private static final Long ONE_DAY_MILLIS = 86400000L; public ServerTablesShardingDateTypeAlgorithm(int tableNums){ this.tableNums = tableNums; } private String getModValue(Date value){ long days = DateUtil.getAfterUnixDays(value); return "_" + (days & (tableNums - 1)); } @Override public String doEqualSharding(Collection<String> availableTargetNames, ShardingValue<Date> shardingValue) { String tableSuffix = getModValue(shardingValue.getValue()); for (String eachTableName : availableTargetNames){ if (eachTableName.endsWith(tableSuffix)){ return eachTableName; } } throw new IllegalArgumentException(); } @Override public Collection<String> doInSharding(Collection<String> availableTargetNames, ShardingValue<Date> shardingValue) { Collection<String> result = new LinkedHashSet<>(availableTargetNames.size()); for (Date value : shardingValue.getValues()){ String tableSuffix = getModValue(value); for (String eachTableName : availableTargetNames){ if (eachTableName.endsWith(tableSuffix)){ result.add(eachTableName); break; } } } if (result.isEmpty()){ throw new IllegalArgumentException(); } return result; } @Override public Collection<String> doBetweenSharding(Collection<String> availableTargetNames, ShardingValue<Date> shardingValue) { Collection<String> result = new LinkedHashSet<>(availableTargetNames.size()); Range<Date> dateRange = shardingValue.getValueRange(); Date endDate = dateRange.upperEndpoint(); for (Date value = dateRange.lowerEndpoint() ; value.getTime() <= endDate.getTime() ; value = DateUtil.addDays(value,1)){ String tableSuffix = getModValue(value); for (String eachTableName : availableTargetNames){ if (eachTableName.endsWith(tableSuffix)){ result.add(eachTableName); break; } } } Long tmp = endDate.getTime() - dateRange.lowerEndpoint().getTime(); if (tmp < ONE_DAY_MILLIS){ tmp = tmp / 1000; for (Date value = dateRange.lowerEndpoint(); value.getTime() <= endDate.getTime() ; value = DateUtil.addSeconds(value,tmp.intValue())){ String tableSuffix = getModValue(value); for (String eachTableName : availableTargetNames){ if (eachTableName.endsWith(tableSuffix)){ if (!result.contains(eachTableName)){ result.add(eachTableName); } break; } } } } if (result.isEmpty()){ throw new IllegalArgumentException(); } return result; } }
package com.test.yuhua.common.constant; public enum Target { // 机器 server, // 机房 node, ; public static Target get(final String trgt){ Target target; target = Target.valueOf(trgt.toLowerCase()); return target; } }
package com.test.yuhua.common.constant; public enum Purpose { // 历史 history, // 趋势 trend, ; public static Purpose get(final String prps) { Purpose purpose; purpose = Purpose.valueOf(prps.toLowerCase()); return purpose; } }
package com.test.yuhua.service.config.mybatis; import com.dangdang.ddframe.rdb.sharding.api.rule.DataSourceRule; import com.dangdang.ddframe.rdb.sharding.api.rule.ShardingRule; import com.dangdang.ddframe.rdb.sharding.api.rule.TableRule; import com.dangdang.ddframe.rdb.sharding.api.strategy.database.DatabaseShardingStrategy; import com.dangdang.ddframe.rdb.sharding.api.strategy.database.NoneDatabaseShardingAlgorithm; import com.dangdang.ddframe.rdb.sharding.api.strategy.table.TableShardingStrategy; import com.dangdang.ddframe.rdb.sharding.jdbc.ShardingDataSource; import com.test.yuhua.common.constant.Purpose; import com.test.yuhua.common.constant.Target; import com.test.yuhua.service.dynamic.shard.DynamicDataSource; import com.test.yuhua.service.dynamic.shard.ServerTablesShardingDateTypeAlgorithm; import org.springframework.context.annotation.Bean; import javax.sql.DataSource; import java.text.MessageFormat; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; public abstract class AbstractBussinessConfig extends MyBatisBaseConfig { abstract Map<Object,Object> setTargetSources(); @Bean public DynamicDataSource dynamicDataSource(){ DynamicDataSource source = new DynamicDataSource(); source.setTargetDataSources(setTargetSources()); return source; } // 服务器历史数据表分表个数 private static final int TABLE_NUMS_16 = 16; private static final String[] SHARDING_ENTITY_NAMES = {"retcode", "slowspeed", "delaydist", "requestdelay", "bitframe", "gameqos_retcode", "gameqos_delaydist", "squidqos", "delay", "cachehit", "bandwidthspeed"}; private static final String TABLE_NAME_FORMATE = "{0}_{1}_{2}"; private static String getTableName(Target trgt, Purpose prps, String parentEntity) { return MessageFormat.format(TABLE_NAME_FORMATE, trgt, parentEntity, prps); } DataSource getDangDangShardingDataSource(DataSource dataSource){ DataSourceRule dataSourceRule = getDataSourceRule(dataSource); List<TableRule> rules = new ArrayList<>(); getSixteenTablesShardingRule(dataSourceRule,rules); return getShardingDataSource(dataSourceRule, rules); } private DataSourceRule getDataSourceRule(DataSource dataSource){ Map<String,DataSource> targetDataSources = new HashMap<>(1); targetDataSources.put("dynamic",dataSource); return new DataSourceRule(targetDataSources); } private void getSixteenTablesShardingRule(DataSourceRule dataSourceRule,List<TableRule> tableRules){ TableShardingStrategy tableShardingStrategy = getTablesShardingStrategy(TABLE_NUMS_16); for (String entityName : SHARDING_ENTITY_NAMES){ for (Purpose purpose : Purpose.values()){ String logicTable = getTableName(Target.server,purpose,entityName); tableRules.add(getServerTableRule(dataSourceRule,tableShardingStrategy,logicTable,TABLE_NUMS_16)); } } } private TableShardingStrategy getTablesShardingStrategy(int tableNums){ return new TableShardingStrategy("clock",new ServerTablesShardingDateTypeAlgorithm(tableNums)); } private TableRule getServerTableRule(DataSourceRule dataSourceRule,TableShardingStrategy strategy,String logicTable,int tableNums){ List<String> allTables = getAllTables(logicTable,tableNums); return TableRule.builder(logicTable).actualTables(allTables).dataSourceRule(dataSourceRule).tableShardingStrategy(strategy).build(); } private List<String> getAllTables(String tableName,int number){ String tablePrefix = tableName.concat("_"); List<String> list = new ArrayList<>(number); for (int i = 0 ; i < number ; i++){ list.add(tablePrefix + i); } return list; } private DataSource getShardingDataSource(DataSourceRule dataSourceRule,List<TableRule> rules){ ShardingRule.ShardingRuleBuilder shardingRuleBuilder = getShardingBuilder(dataSourceRule); shardingRuleBuilder.tableRules(rules); return new ShardingDataSource(shardingRuleBuilder.build()); } private ShardingRule.ShardingRuleBuilder getShardingBuilder(DataSourceRule dataSourceRule){ return ShardingRule.builder().dataSourceRule(dataSourceRule).databaseShardingStrategy(getDatabaseShardingStrategy()); } private DatabaseShardingStrategy getDatabaseShardingStrategy(){ return new DatabaseShardingStrategy("none",new NoneDatabaseShardingAlgorithm()); } }
package com.test.yuhua.service.config.mybatis; import com.test.yuhua.common.constant.Platform; import org.apache.ibatis.plugin.Interceptor; import org.apache.ibatis.session.SqlSessionFactory; import org.mybatis.spring.SqlSessionFactoryBean; import org.mybatis.spring.annotation.MapperScan; import org.springframework.boot.autoconfigure.jdbc.DataSourceBuilder; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import javax.sql.DataSource; import java.util.HashMap; import java.util.Map; @Configuration @MapperScan(basePackages = "com.test.yuhua.service.slave.dao", sqlSessionFactoryRef = "slaveSqlSessionFactory") public class MyBatisSlaveConfig extends AbstractBussinessConfig { @Bean(name = "slaveSqlSessionFactory") public SqlSessionFactory sqlSessionFactory(){ SqlSessionFactoryBean sqlSessionFactorybean = new SqlSessionFactoryBean(); sqlSessionFactorybean.setDataSource(dynamicDataSource()); sqlSessionFactorybean.setTypeAliasesPackage("com.test.yuhua.service.dao.entity.bussiness"); sqlSessionFactorybean.setPlugins(new Interceptor[]{getPageHelper()}); try { return sqlSessionFactorybean.getObject(); } catch (Exception e) { throw new RuntimeException(e); } } @Override Map<Object, Object> setTargetSources() { Map<Object,Object> targetDataSources = new HashMap<>(Platform.values().length); targetDataSources.put(Platform.live2,getDangDangShardingDataSource(slaveLive2DataSource())); return targetDataSources; } @Bean(initMethod = "init", destroyMethod = "close") @ConfigurationProperties(prefix = "spring.slave.live2.datasource") public DataSource slaveLive2DataSource(){ return DataSourceBuilder.create().type(dataSourceType).build(); } }
☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆
☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆
☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆
package com.test.yuhua.common.constant; import lombok.AllArgsConstructor; import lombok.Getter; @Getter @AllArgsConstructor public enum CmdbAppCode { CENTER("cachecenter","pswn-cae"), BORDER("cache","pswn-cah"); private final String name; private final String desc; public static CmdbAppCode getByName(String name){ if ("cachecenter".equals(name)){ return CENTER; }else if ("cache".equals(name)){ return BORDER; } throw new RuntimeException(); } }
package com.test.yuhua.service.config.mybatis; import com.test.yuhua.common.constant.Platform; import org.apache.ibatis.plugin.Interceptor; import org.apache.ibatis.session.SqlSessionFactory; import org.mybatis.spring.SqlSessionFactoryBean; import org.mybatis.spring.annotation.MapperScan; import org.springframework.boot.autoconfigure.jdbc.DataSourceBuilder; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import javax.sql.DataSource; import java.util.HashMap; import java.util.Map; @Configuration @MapperScan(basePackages = "com.test.yuhua.service.slave.dao", sqlSessionFactoryRef = "slaveSqlSessionFactory") public class MyBatisSlaveConfig extends AbstractBussinessConfig { @Bean(name = "slaveSqlSessionFactory") public SqlSessionFactory sqlSessionFactory(){ SqlSessionFactoryBean sqlSessionFactorybean = new SqlSessionFactoryBean(); sqlSessionFactorybean.setDataSource(dynamicDataSource()); sqlSessionFactorybean.setTypeAliasesPackage("com.test.yuhua.service.dao.entity.bussiness"); sqlSessionFactorybean.setPlugins(new Interceptor[]{getPageHelper()}); try { return sqlSessionFactorybean.getObject(); } catch (Exception e) { throw new RuntimeException(e); } } @Override Map<Object, Object> setTargetSources() { Map<Object,Object> targetDataSources = new HashMap<>(Platform.values().length); targetDataSources.put(Platform.live2,getDangDangShardingDataSource(slaveLive2DataSource())); targetDataSources.put(Platform.p2sp,getP2spShardingDataSource(slaveP2spDataSource())); targetDataSources.put(Platform.webcdn, getWebCdnShardingDataSource(slaveWebCdnDataSource())); return targetDataSources; } @Bean(initMethod = "init", destroyMethod = "close") @ConfigurationProperties(prefix = "spring.slave.live2.datasource") public DataSource slaveLive2DataSource(){ return DataSourceBuilder.create().type(dataSourceType).build(); } @Bean(initMethod = "init", destroyMethod = "close") @ConfigurationProperties(prefix = "spring.slave.p2sp.datasource") public DataSource slaveP2spDataSource() { return DataSourceBuilder.create().type(dataSourceType).build(); } @Bean(initMethod = "init", destroyMethod = "close") @ConfigurationProperties(prefix = "spring.slave.webcdn.datasource") public DataSource slaveWebCdnDataSource() { return DataSourceBuilder.create().type(dataSourceType).build(); } }
package com.test.yuhua.service.config.mybatis; import com.dangdang.ddframe.rdb.sharding.api.rule.DataSourceRule; import com.dangdang.ddframe.rdb.sharding.api.rule.ShardingRule; import com.dangdang.ddframe.rdb.sharding.api.rule.TableRule; import com.dangdang.ddframe.rdb.sharding.api.strategy.database.DatabaseShardingStrategy; import com.dangdang.ddframe.rdb.sharding.api.strategy.database.NoneDatabaseShardingAlgorithm; import com.dangdang.ddframe.rdb.sharding.api.strategy.table.NoneTableShardingAlgorithm; import com.dangdang.ddframe.rdb.sharding.api.strategy.table.TableShardingStrategy; import com.dangdang.ddframe.rdb.sharding.jdbc.ShardingDataSource; import com.test.yuhua.common.constant.CmdbAppCode; import com.test.yuhua.common.constant.Purpose; import com.test.yuhua.common.constant.Target; import com.test.yuhua.service.dynamic.shard.DynamicDataSource; import com.test.yuhua.service.dynamic.shard.ServerTablesShardingDateTypeAlgorithm; import org.springframework.context.annotation.Bean; import javax.sql.DataSource; import java.text.MessageFormat; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; public abstract class AbstractBussinessConfig extends MyBatisBaseConfig { abstract Map<Object,Object> setTargetSources(); @Bean public DynamicDataSource dynamicDataSource(){ DynamicDataSource source = new DynamicDataSource(); source.setTargetDataSources(setTargetSources()); return source; } // 服务器历史数据表分表个数 private static final int TABLE_NUMS_16 = 16; private static final int TABLE_NUMS_4 = 4; private static final int TABLE_NUMS_8 = 8; private static final int TABLE_NUMS_256 = 256; private static final String[] SHARDING_ENTITY_NAMES = {"retcode", "slowspeed", "delaydist", "requestdelay", "bitframe", "gameqos_retcode", "gameqos_delaydist", "squidqos", "delay", "cachehit", "bandwidthspeed"}; private static final String TABLE_NAME_FORMATE = "{0}_{1}_{2}"; private static String getTableName(Target trgt, Purpose prps, String parentEntity) { return MessageFormat.format(TABLE_NAME_FORMATE, trgt, parentEntity, prps); } private static String getTableName(CmdbAppCode appCode, Purpose prps, String parentEntity) { return MessageFormat.format(TABLE_NAME_FORMATE, appCode.name().toLowerCase(), parentEntity, prps); } /** * Platform.live2 */ DataSource getDangDangShardingDataSource(DataSource dataSource){ DataSourceRule dataSourceRule = getDataSourceRule(dataSource); List<TableRule> rules = new ArrayList<>(); getSixteenTablesShardingRule(dataSourceRule,rules); getFourTablesShardingRule(dataSourceRule,rules); get255TablesShardingRule(dataSourceRule,rules); return getShardingDataSource(dataSourceRule, rules); } private DataSourceRule getDataSourceRule(DataSource dataSource){ Map<String,DataSource> targetDataSources = new HashMap<>(1); targetDataSources.put("dynamic",dataSource); return new DataSourceRule(targetDataSources); } private void getSixteenTablesShardingRule(DataSourceRule dataSourceRule,List<TableRule> tableRules){ TableShardingStrategy tableShardingStrategy = getTablesShardingStrategy(TABLE_NUMS_16); for (String entityName : SHARDING_ENTITY_NAMES){ for (Purpose purpose : Purpose.values()){ String logicTable = getTableName(Target.server,purpose,entityName); tableRules.add(getServerTableRule(dataSourceRule,tableShardingStrategy,logicTable,TABLE_NUMS_16)); } } } private void getFourTablesShardingRule(DataSourceRule dataSourceRule,List<TableRule> tableRules){ TableShardingStrategy tableShardingStrategy = getTablesShardingStrategy(TABLE_NUMS_4); tableRules.add(getServerTableRule(dataSourceRule,tableShardingStrategy,getTableName(Target.server,Purpose.history,"gameqos_client"),TABLE_NUMS_4)); tableRules.add(getServerTableRule(dataSourceRule,tableShardingStrategy,"alarm",TABLE_NUMS_4)); tableRules.add(getServerTableRule(dataSourceRule,tableShardingStrategy,"alarm_data",TABLE_NUMS_4)); } private void get255TablesShardingRule(DataSourceRule dataSourceRule,List<TableRule> tableRules){ TableShardingStrategy tableShardingStrategy = getTablesShardingStrategy(TABLE_NUMS_256); tableRules.add(getServerTableRule(dataSourceRule,tableShardingStrategy,"item_data",TABLE_NUMS_256)); } private TableShardingStrategy getTablesShardingStrategy(int tableNums){ return new TableShardingStrategy("clock",new ServerTablesShardingDateTypeAlgorithm(tableNums)); } private TableRule getServerTableRule(DataSourceRule dataSourceRule,TableShardingStrategy strategy,String logicTable,int tableNums){ List<String> allTables = getAllTables(logicTable,tableNums); return TableRule.builder(logicTable).actualTables(allTables).dataSourceRule(dataSourceRule).tableShardingStrategy(strategy).build(); } private List<String> getAllTables(String tableName,int number){ String tablePrefix = tableName.concat("_"); List<String> list = new ArrayList<>(number); for (int i = 0 ; i < number ; i++){ list.add(tablePrefix + i); } return list; } private DataSource getShardingDataSource(DataSourceRule dataSourceRule,List<TableRule> rules){ ShardingRule.ShardingRuleBuilder shardingRuleBuilder = getShardingBuilder(dataSourceRule); shardingRuleBuilder.tableRules(rules); return new ShardingDataSource(shardingRuleBuilder.build()); } private ShardingRule.ShardingRuleBuilder getShardingBuilder(DataSourceRule dataSourceRule){ return ShardingRule.builder().dataSourceRule(dataSourceRule).databaseShardingStrategy(getDatabaseShardingStrategy()); } private DatabaseShardingStrategy getDatabaseShardingStrategy(){ return new DatabaseShardingStrategy("none",new NoneDatabaseShardingAlgorithm()); } /** * Platform.p2sp */ DataSource getP2spShardingDataSource(DataSource dataSource){ DataSourceRule dataSourceRule = getDataSourceRule(dataSource); List<TableRule> rules = new ArrayList<TableRule>(); getP2spEightTablesShardingRule(dataSourceRule,rules); return getShardingDataSource(dataSourceRule,rules); } private static final String[] P2SP_SHARDING_8_TABLES_NAMES = {"cdnqos_history", "p2pqos_history","bwspareratio_history"}; private void getP2spEightTablesShardingRule(DataSourceRule dataSourceRule,List<TableRule> tableRules){ TableShardingStrategy tableShardingStrategy = getTablesShardingStrategy(TABLE_NUMS_8); for (String logicTable : P2SP_SHARDING_8_TABLES_NAMES){ tableRules.add(getServerTableRule(dataSourceRule,tableShardingStrategy,logicTable,TABLE_NUMS_8)); } } /** * Platform.webcdn */ DataSource getWebCdnShardingDataSource(DataSource dataSource){ DataSourceRule dataSourceRule = getDataSourceRule(dataSource); List<TableRule> tableRules = new ArrayList<TableRule>(); getSixteenTablesShardingRule(dataSourceRule,tableRules); getWebCdnEightTablesShardingRule(dataSourceRule,tableRules); getFourTablesShardingRule(dataSourceRule,tableRules); getOneTableShardingRule(dataSourceRule,tableRules); get255TablesShardingRule(dataSourceRule,tableRules); return getShardingDataSource(dataSourceRule,tableRules); } private static final String[] DOMAIN_SHARDING_ENTITY_NAMES = {"domain_retcode", "domain_bandwidthspeed", "domain_delay", "domain_cachehit"}; private void getWebCdnEightTablesShardingRule(DataSourceRule dataSourceRule,List<TableRule> tableRules){ TableShardingStrategy tableShardingStrategy = getTablesShardingStrategy(TABLE_NUMS_8); for (String entityName : DOMAIN_SHARDING_ENTITY_NAMES){ for (CmdbAppCode appCode : CmdbAppCode.values()){ String logicTable = getTableName(appCode,Purpose.history,entityName); tableRules.add(getServerTableRule(dataSourceRule,tableShardingStrategy,logicTable,TABLE_NUMS_8)); } } } private void getOneTableShardingRule(DataSourceRule dataSourceRule,List<TableRule> tableRules){ TableShardingStrategy tableShardingStrategy = new TableShardingStrategy("",new NoneTableShardingAlgorithm()); tableRules.add(TableRule.builder("domain").dataSourceRule(dataSourceRule).tableShardingStrategy(tableShardingStrategy).build()); } }