1. Jmeter的使用
(1)jmeter 到底是干啥的请看:https://blog.csdn.net/zl1zl2zl3/article/details/78194194
(2)jmeter 压力测试报告参数的具体含义请看:https://blog.csdn.net/u012111923/article/details/80705141
注:我测试使用的jmeter为2018-11-6最新的5.X,但是大同小异。
2.Jmeter接口程序编写
package jmeter;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.jmeter.config.Arguments;
import org.apache.jmeter.protocol.java.sampler.JavaSamplerClient;
import org.apache.jmeter.protocol.java.sampler.JavaSamplerContext;
import org.apache.jmeter.samplers.SampleResult;
import java.io.BufferedReader;
import java.io.FileReader;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import java.util.Random;
public class MyHbaseSampler implements JavaSamplerClient {
HTable ht = null;
Random rnd = new Random();
String XPath;
private static Map<Integer, String> cacheMap = new HashMap<Integer, String>();
@Override
public void setupTest(JavaSamplerContext context) {
Configuration HBASE_CONFIG = new Configuration();
HBASE_CONFIG.set("hbase.zookeeper.quorum", context.getParameter("zk.hosts"));//zookeeper
HBASE_CONFIG.set("hbase.zookeeper.property.clientPort", context.getParameter("zk.port"));
HBASE_CONFIG.set("zookeeper.znode.parent", context.getParameter("zk.dir"));
HBASE_CONFIG.setLong("zookeeper.session.timeout", 900000);
HBASE_CONFIG.setInt("hbase.client.retries.number", 5);
HBASE_CONFIG.setInt("hbase.meta.scanner.caching", 5000);
HBASE_CONFIG.setInt("hbase.client.prefetch.limit", 100);
Configuration configuration = HBaseConfiguration.create(HBASE_CONFIG);
try {
ht = new HTable(configuration, context.getParameter("hbase.table"));
ht.setWriteBufferSize(8 * 1024 * 1024);
ht.setAutoFlush(false, false);
} catch (IOException e) {
e.printStackTrace();
}
XPath = context.getParameter("XPath");
int i = 0;
try {
BufferedReader br = new BufferedReader(new FileReader(XPath));//构造一个BufferedReader类来读取文件
String s = null;
while ((s = br.readLine()) != null) {//使用readLine方法,一次读一行
cacheMap.put(i, s);
i += 1;
}
br.close();
} catch (IOException e) {
e.printStackTrace();
}
}
@Override
public SampleResult runTest(JavaSamplerContext context) {
SampleResult result = new SampleResult();
try {
result.sampleStart();
int mapKey = rnd.nextInt(100000000);
String mapValue = cacheMap.get(mapKey);
byte[] rk = Bytes.add(Bytes.toBytes((short) (mapValue.hashCode() & 0x7fff)), Bytes.toBytes(mapValue));
Get get = new Get(rk);
Result result1 = ht.get(get);
/*Cell[] cells = result1.rawCells();
for (Cell cell : cells) {
System.out.println("RowName:" + new String(CellUtil.cloneRow(cell)) + " ");
System.out.println("Timetamp:" + cell.getTimestamp() + " ");
System.out.println("column Family:" + new String(CellUtil.cloneFamily(cell)) + " ");
System.out.println("row Name:" + new String(CellUtil.cloneQualifier(cell)) + " ");
System.out.println("value:" + new String(CellUtil.cloneValue(cell),"UTF-8") + " ");
}*/
result.setBodySize(result1.getRow().length);
result.setSuccessful(true);
} catch (Exception e) {
result.setSuccessful(false);
} finally {
result.sampleEnd();
}
return result;
}
@Override
public void teardownTest(JavaSamplerContext context) {
try {
cacheMap.clear();
ht.close();
} catch (IOException e) {
e.printStackTrace();
}
}
@Override
public Arguments getDefaultParameters() {
Arguments args = new Arguments();
args.addArgument("zk.hosts", "*.*.*.*");
args.addArgument("zk.port", "2181");
args.addArgument("zk.dir", "/hbase");
args.addArgument("hbase.table", "test_www");
//args.addArgument("math.random", "6000");
args.addArgument("XPath", "/home/hadoop/uniq_info_number.txt");
return args;
}
public static void main(String[] args) {
System.out.println("w");
}
}
jmeter测试hbase的样例程序如上,功能是:读取/home/hadoop/uniq_info_number.txt(1亿个不同的手机号),然后缓存到map中,形式<0,一行数据>,<1,下一行数据>.......
注意:(1)先在windows下进行:
这块代码直接粘贴到eclipse中,然后eclipse导入jmeter的jar包(只导入lib/ext/ApacheJmeter包)
java写上main方法,运行后,直接export--》runnable jar,然后最重要的一步是把生成的get.jar(名字任意)放入到jmeter/lib/ext中,这个时候打开cmd,然后输入jmeter.bat,等待jmeter打开,观察cmd上是否有错误出现,同时点击jmeter,右击添加 -> thread group -> sample -> java request。观察是否识别到你的自定义测试类正常如下:
然后点击保存到桌面java.jmx,windows下的操作至此完毕。
注:加载jar包时会在cmd中报错,out of memory,修改方法是:编辑jmeter.bat,找到
set HEAP=-Xms1g -Xmx1g -XX:MaxMetaspaceSize=256m
修改为:
set HEAP=-Xms3g -Xmx3g -XX:MaxMetaspaceSize=1024m
(2)第二步是linux下的操作:
下载tar.gz包,然后解压缩,配置环境变量,为了防止出现out of memory,直接修改bin/jmeter(注仅仅是jmeter这个文件)
找到:
: "${HEAP:="-Xms1g -Xmx1g -XX:MaxMetaspaceSize=256m"}"
修改为:
: "${HEAP:="-Xms20g -Xmx20g -XX:MaxMetaspaceSize=5g"}"
然后上传get.jar和java.jmx,然后把get.jar放到jmeter/lib/ext下,每次均要修改java.jmx的两个参数进行测试:如下
num.threads指定并发数,loops指定每个并发访问的次数,两者相乘就是总的请求数。
运行指令如下:
jmeter -n -t java.jmx -l listener.csv
程序运行后会生成数据文件listener.csv,数据条数为请求数,自动生成jmeter.log,可查看日志是否有报错,然后把生成的数据文件放到windos下,在jmeter线程组中添加聚合图,导入数据文件,查看分析结果。
注意:每次导入文件后,查看记录数据后,都要清空结果,然后重新导入下批文件。
测试结果如下:不同的并发,均访问100次
3 测试过程注意事项:
(1)如果加hbase节点,要执行balancer,平行region
(2)程序入库后要大合并: major_compact "test_www"
(3)读测试要开启bloom过滤:alter 'test_www', {NAME => 'f',BLOOMFILTER => 'ROWCOL'}
(4)每次测试完之后要重启hbase库(重启进程),重新major_compact,不然hbase会缓存一些数据,造成下次的jmeter测试命中率更高,结果不准确。
(5)修改hbase-site.xml更利于读:
<property>
<name>hbase.regionserver.global.memstore.upperLimit</name>
<value>0.4</value>
</property>
<property>
<name>hfile.block.cache.size</name>
<value>0.4</value>
</property>
4 附带jmeter测试redis
import org.apache.jmeter.config.Arguments;
import org.apache.jmeter.protocol.java.sampler.JavaSamplerClient;
import org.apache.jmeter.protocol.java.sampler.JavaSamplerContext;
import org.apache.jmeter.samplers.SampleResult;
import redis.clients.jedis.Jedis;
import redis.clients.jedis.JedisPool;
import redis.clients.jedis.JedisPoolConfig;
public class MyJavaSampler implements JavaSamplerClient {
public JedisPool _jedisPool;
@Override
public void setupTest(JavaSamplerContext context) {
String host = context.getParameter("redis.host");
int port = context.getIntParameter("redis.port");
_jedisPool = new JedisPool(new JedisPoolConfig(),host,port);
}
@Override
public SampleResult runTest(JavaSamplerContext context) {
Jedis jedis = _jedisPool.getResource();
SampleResult result = new SampleResult();
try {
result.sampleStart();
for (int i = 0; i < 1000; i ++){
jedis.set("foo"+i,"bar"+i);
}
result.setSuccessful(true);
}catch (Exception e){
result.setSuccessful(false);
}finally {
result.sampleEnd();
}
return result;
}
@Override
public void teardownTest(JavaSamplerContext context) {
_jedisPool.close();
}
@Override
public Arguments getDefaultParameters() {
Arguments args = new Arguments();
args.addArgument("redis.host","127.0.0.1");
args.addArgument("redis.port","6379");
return args;
}
}
5 附带手机号为15300000000-15399999999的随机hbase查代码
package jmeter;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.jmeter.config.Arguments;
import org.apache.jmeter.protocol.java.sampler.JavaSamplerClient;
import org.apache.jmeter.protocol.java.sampler.JavaSamplerContext;
import org.apache.jmeter.samplers.SampleResult;
import java.io.IOException;
import java.util.Random;
public class MyHbaseSampler implements JavaSamplerClient {
HTable ht = null;
Random rnd = new Random();
int rang;
public MyHbaseSampler() {
}
public void setupTest(JavaSamplerContext context) {
Configuration HBASE_CONFIG = new Configuration();
HBASE_CONFIG.set("hbase.zookeeper.quorum", context.getParameter("zk.hosts"));
HBASE_CONFIG.set("hbase.zookeeper.property.clientPort", context.getParameter("zk.port"));
HBASE_CONFIG.set("zookeeper.znode.parent", context.getParameter("zk.dir"));
HBASE_CONFIG.setLong("zookeeper.session.timeout", 900000L);
HBASE_CONFIG.setInt("hbase.client.retries.number", 5);
HBASE_CONFIG.setInt("hbase.meta.scanner.caching", 5000);
HBASE_CONFIG.setInt("hbase.client.prefetch.limit", 100);
Configuration configuration = HBaseConfiguration.create(HBASE_CONFIG);
try {
ht = new HTable(configuration, context.getParameter("hbase.table"));
ht.setWriteBufferSize(8388608L);
ht.setAutoFlush(false, false);
} catch (IOException e) {
e.printStackTrace();
}
rang = context.getIntParameter("rang");
}
public SampleResult runTest(JavaSamplerContext context) {
SampleResult result = new SampleResult();
try {
result.sampleStart();
String phoneHead = "15";
int randNum = rnd.nextInt(rang);
String phoneNum = phoneHead + String.format("%09d", new Object[] { Integer.valueOf(randNum) });
byte[] rowdKey = Bytes.add(Bytes.toBytes((short) (phoneNum.hashCode() & 0x7FFF)), Bytes.toBytes(phoneNum));
Get get = new Get(rowdKey);
Result result1 = ht.get(get);
result.setBodySize(result1.getRow().length);
result.setSuccessful(true);
} catch (Exception e) {
result.setSuccessful(false);
} finally {
result.sampleEnd();
}
return result;
}
public void teardownTest(JavaSamplerContext context) {
try {
ht.close();
} catch (IOException e) {
e.printStackTrace();
}
}
public Arguments getDefaultParameters() {
Arguments args = new Arguments();
args.addArgument("zk.hosts", "10.161.48.91,10.161.48.92,10.161.48.93");
args.addArgument("zk.port", "2181");
args.addArgument("zk.dir", "/hbase");
args.addArgument("hbase.table", "test_www");
args.addArgument("rang", "100000000");
return args;
}
public static void main(String[] args) {
System.out.println("w");
}
}