1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17 package org.apache.hadoop.hbase.client;
18
19 import static org.junit.Assert.assertEquals;
20 import static org.junit.Assert.assertTrue;
21
22 import java.util.ArrayList;
23 import java.util.List;
24
25 import org.apache.commons.logging.Log;
26 import org.apache.commons.logging.LogFactory;
27 import org.apache.hadoop.hbase.Cell;
28 import org.apache.hadoop.hbase.HBaseTestingUtility;
29 import org.apache.hadoop.hbase.HRegionInfo;
30 import org.apache.hadoop.hbase.HRegionLocation;
31 import org.apache.hadoop.hbase.HTestConst;
32 import org.apache.hadoop.hbase.KeyValue;
33 import org.apache.hadoop.hbase.testclassification.MediumTests;
34 import org.apache.hadoop.hbase.MiniHBaseCluster;
35 import org.apache.hadoop.hbase.filter.ColumnPrefixFilter;
36 import org.apache.hadoop.hbase.filter.ColumnRangeFilter;
37 import org.apache.hadoop.hbase.master.HMaster;
38 import org.apache.hadoop.hbase.master.RegionState.State;
39 import org.apache.hadoop.hbase.master.RegionStates;
40 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
41 import org.apache.hadoop.hbase.regionserver.HRegionServer;
42 import org.apache.hadoop.hbase.util.Bytes;
43 import org.apache.hadoop.hbase.util.ConfigUtil;
44 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
45 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
46 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
47 import org.junit.After;
48 import org.junit.AfterClass;
49 import org.junit.Before;
50 import org.junit.BeforeClass;
51 import org.junit.Test;
52 import org.junit.experimental.categories.Category;
53
54
55
56
57 @Category(MediumTests.class)
58 public class TestScannersFromClientSide {
59 private static final Log LOG = LogFactory.getLog(TestScannersFromClientSide.class);
60
61 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
62 private static byte [] ROW = Bytes.toBytes("testRow");
63 private static byte [] FAMILY = Bytes.toBytes("testFamily");
64 private static byte [] QUALIFIER = Bytes.toBytes("testQualifier");
65 private static byte [] VALUE = Bytes.toBytes("testValue");
66
67
68
69
70 @BeforeClass
71 public static void setUpBeforeClass() throws Exception {
72 TEST_UTIL.startMiniCluster(3);
73 }
74
75
76
77
78 @AfterClass
79 public static void tearDownAfterClass() throws Exception {
80 TEST_UTIL.shutdownMiniCluster();
81 }
82
83
84
85
86 @Before
87 public void setUp() throws Exception {
88
89 }
90
91
92
93
94 @After
95 public void tearDown() throws Exception {
96
97 }
98
99
100
101
102
103
104 @Test
105 public void testScanBatch() throws Exception {
106 byte [] TABLE = Bytes.toBytes("testScanBatch");
107 byte [][] QUALIFIERS = HTestConst.makeNAscii(QUALIFIER, 8);
108
109 HTable ht = TEST_UTIL.createTable(TABLE, FAMILY);
110
111 Put put;
112 Scan scan;
113 Delete delete;
114 Result result;
115 ResultScanner scanner;
116 boolean toLog = true;
117 List<Cell> kvListExp;
118
119
120 put = new Put(ROW);
121 for (int i=0; i < QUALIFIERS.length; i++) {
122 KeyValue kv = new KeyValue(ROW, FAMILY, QUALIFIERS[i], i, VALUE);
123 put.add(kv);
124 }
125 ht.put(put);
126
127
128 put = new Put(ROW);
129 KeyValue kv = new KeyValue(ROW, FAMILY, QUALIFIERS[6], 2, VALUE);
130 put.add(kv);
131 ht.put(put);
132
133
134 delete = new Delete(ROW);
135 delete.deleteFamily(FAMILY, 3);
136 ht.delete(delete);
137
138
139 scan = new Scan(ROW);
140 scan.setMaxVersions();
141 scanner = ht.getScanner(scan);
142
143
144 kvListExp = new ArrayList<Cell>();
145 kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[4], 4, VALUE));
146 kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[5], 5, VALUE));
147 kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[6], 6, VALUE));
148 kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[7], 7, VALUE));
149 result = scanner.next();
150 verifyResult(result, kvListExp, toLog, "Testing first batch of scan");
151
152
153 scan = new Scan(ROW);
154 scan.setMaxVersions();
155 scan.setBatch(2);
156 scanner = ht.getScanner(scan);
157
158
159 kvListExp = new ArrayList<Cell>();
160 kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[4], 4, VALUE));
161 kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[5], 5, VALUE));
162 result = scanner.next();
163 verifyResult(result, kvListExp, toLog, "Testing first batch of scan");
164
165
166 kvListExp = new ArrayList<Cell>();
167 kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[6], 6, VALUE));
168 kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[7], 7, VALUE));
169 result = scanner.next();
170 verifyResult(result, kvListExp, toLog, "Testing second batch of scan");
171
172 }
173
174
175
176
177
178
179 @Test
180 public void testGetMaxResults() throws Exception {
181 byte [] TABLE = Bytes.toBytes("testGetMaxResults");
182 byte [][] FAMILIES = HTestConst.makeNAscii(FAMILY, 3);
183 byte [][] QUALIFIERS = HTestConst.makeNAscii(QUALIFIER, 20);
184
185 HTable ht = TEST_UTIL.createTable(TABLE, FAMILIES);
186
187 Get get;
188 Put put;
189 Result result;
190 boolean toLog = true;
191 List<Cell> kvListExp;
192
193 kvListExp = new ArrayList<Cell>();
194
195 put = new Put(ROW);
196 for (int i=0; i < 10; i++) {
197 KeyValue kv = new KeyValue(ROW, FAMILIES[0], QUALIFIERS[i], 1, VALUE);
198 put.add(kv);
199 kvListExp.add(kv);
200 }
201 ht.put(put);
202
203 get = new Get(ROW);
204 result = ht.get(get);
205 verifyResult(result, kvListExp, toLog, "Testing without setting maxResults");
206
207 get = new Get(ROW);
208 get.setMaxResultsPerColumnFamily(2);
209 result = ht.get(get);
210 kvListExp = new ArrayList<Cell>();
211 kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[0], 1, VALUE));
212 kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[1], 1, VALUE));
213 verifyResult(result, kvListExp, toLog, "Testing basic setMaxResults");
214
215
216 get = new Get(ROW);
217 get.setMaxResultsPerColumnFamily(5);
218 get.setFilter(new ColumnRangeFilter(QUALIFIERS[2], true, QUALIFIERS[5],
219 true));
220 result = ht.get(get);
221 kvListExp = new ArrayList<Cell>();
222 kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[2], 1, VALUE));
223 kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[3], 1, VALUE));
224 kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[4], 1, VALUE));
225 kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[5], 1, VALUE));
226 verifyResult(result, kvListExp, toLog, "Testing single CF with CRF");
227
228
229
230 put = new Put(ROW);
231 for (int i=0; i < QUALIFIERS.length; i++) {
232 KeyValue kv = new KeyValue(ROW, FAMILIES[2], QUALIFIERS[i], 1, VALUE);
233 put.add(kv);
234 }
235 ht.put(put);
236
237 put = new Put(ROW);
238 for (int i=0; i < 10; i++) {
239 KeyValue kv = new KeyValue(ROW, FAMILIES[1], QUALIFIERS[i], 1, VALUE);
240 put.add(kv);
241 }
242 ht.put(put);
243
244 get = new Get(ROW);
245 get.setMaxResultsPerColumnFamily(12);
246 get.addFamily(FAMILIES[1]);
247 get.addFamily(FAMILIES[2]);
248 result = ht.get(get);
249 kvListExp = new ArrayList<Cell>();
250
251 for (int i=0; i < 10; i++) {
252 kvListExp.add(new KeyValue(ROW, FAMILIES[1], QUALIFIERS[i], 1, VALUE));
253 }
254 for (int i=0; i < 2; i++) {
255 kvListExp.add(new KeyValue(ROW, FAMILIES[2], QUALIFIERS[i], 1, VALUE));
256 }
257 for (int i=10; i < 20; i++) {
258 kvListExp.add(new KeyValue(ROW, FAMILIES[2], QUALIFIERS[i], 1, VALUE));
259 }
260 verifyResult(result, kvListExp, toLog, "Testing multiple CFs");
261
262
263 get = new Get(ROW);
264 get.setMaxResultsPerColumnFamily(3);
265 get.setFilter(new ColumnRangeFilter(QUALIFIERS[2], true, null, true));
266 result = ht.get(get);
267 kvListExp = new ArrayList<Cell>();
268 for (int i=2; i < 5; i++) {
269 kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[i], 1, VALUE));
270 }
271 for (int i=2; i < 5; i++) {
272 kvListExp.add(new KeyValue(ROW, FAMILIES[1], QUALIFIERS[i], 1, VALUE));
273 }
274 for (int i=2; i < 5; i++) {
275 kvListExp.add(new KeyValue(ROW, FAMILIES[2], QUALIFIERS[i], 1, VALUE));
276 }
277 verifyResult(result, kvListExp, toLog, "Testing multiple CFs + CRF");
278
279 get = new Get(ROW);
280 get.setMaxResultsPerColumnFamily(7);
281 get.setFilter(new ColumnPrefixFilter(QUALIFIERS[1]));
282 result = ht.get(get);
283 kvListExp = new ArrayList<Cell>();
284 kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[1], 1, VALUE));
285 kvListExp.add(new KeyValue(ROW, FAMILIES[1], QUALIFIERS[1], 1, VALUE));
286 kvListExp.add(new KeyValue(ROW, FAMILIES[2], QUALIFIERS[1], 1, VALUE));
287 for (int i=10; i < 16; i++) {
288 kvListExp.add(new KeyValue(ROW, FAMILIES[2], QUALIFIERS[i], 1, VALUE));
289 }
290 verifyResult(result, kvListExp, toLog, "Testing multiple CFs + PFF");
291
292 }
293
294
295
296
297
298
299 @Test
300 public void testScanMaxResults() throws Exception {
301 byte [] TABLE = Bytes.toBytes("testScanLimit");
302 byte [][] ROWS = HTestConst.makeNAscii(ROW, 2);
303 byte [][] FAMILIES = HTestConst.makeNAscii(FAMILY, 3);
304 byte [][] QUALIFIERS = HTestConst.makeNAscii(QUALIFIER, 10);
305
306 HTable ht = TEST_UTIL.createTable(TABLE, FAMILIES);
307
308 Put put;
309 Scan scan;
310 Result result;
311 boolean toLog = true;
312 List<Cell> kvListExp, kvListScan;
313
314 kvListExp = new ArrayList<Cell>();
315
316 for (int r=0; r < ROWS.length; r++) {
317 put = new Put(ROWS[r]);
318 for (int c=0; c < FAMILIES.length; c++) {
319 for (int q=0; q < QUALIFIERS.length; q++) {
320 KeyValue kv = new KeyValue(ROWS[r], FAMILIES[c], QUALIFIERS[q], 1, VALUE);
321 put.add(kv);
322 if (q < 4) {
323 kvListExp.add(kv);
324 }
325 }
326 }
327 ht.put(put);
328 }
329
330 scan = new Scan();
331 scan.setMaxResultsPerColumnFamily(4);
332 ResultScanner scanner = ht.getScanner(scan);
333 kvListScan = new ArrayList<Cell>();
334 while ((result = scanner.next()) != null) {
335 for (Cell kv : result.listCells()) {
336 kvListScan.add(kv);
337 }
338 }
339 result = Result.create(kvListScan);
340 verifyResult(result, kvListExp, toLog, "Testing scan with maxResults");
341
342 }
343
344
345
346
347
348
349 @Test
350 public void testGetRowOffset() throws Exception {
351 byte [] TABLE = Bytes.toBytes("testGetRowOffset");
352 byte [][] FAMILIES = HTestConst.makeNAscii(FAMILY, 3);
353 byte [][] QUALIFIERS = HTestConst.makeNAscii(QUALIFIER, 20);
354
355 HTable ht = TEST_UTIL.createTable(TABLE, FAMILIES);
356
357 Get get;
358 Put put;
359 Result result;
360 boolean toLog = true;
361 List<Cell> kvListExp;
362
363
364 kvListExp = new ArrayList<Cell>();
365 put = new Put(ROW);
366 for (int i=0; i < 10; i++) {
367 KeyValue kv = new KeyValue(ROW, FAMILIES[0], QUALIFIERS[i], 1, VALUE);
368 put.add(kv);
369
370 if (i < 2) continue;
371 kvListExp.add(kv);
372 }
373 ht.put(put);
374
375
376 get = new Get(ROW);
377 get.setRowOffsetPerColumnFamily(2);
378 result = ht.get(get);
379 verifyResult(result, kvListExp, toLog, "Testing basic setRowOffset");
380
381
382 get = new Get(ROW);
383 get.setRowOffsetPerColumnFamily(20);
384 result = ht.get(get);
385 kvListExp = new ArrayList<Cell>();
386 verifyResult(result, kvListExp, toLog, "Testing offset > #kvs");
387
388
389 get = new Get(ROW);
390 get.setRowOffsetPerColumnFamily(4);
391 get.setMaxResultsPerColumnFamily(5);
392 result = ht.get(get);
393 kvListExp = new ArrayList<Cell>();
394 for (int i=4; i < 9; i++) {
395 kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[i], 1, VALUE));
396 }
397 verifyResult(result, kvListExp, toLog,
398 "Testing offset + setMaxResultsPerCF");
399
400
401 get = new Get(ROW);
402 get.setRowOffsetPerColumnFamily(1);
403 get.setFilter(new ColumnRangeFilter(QUALIFIERS[2], true, QUALIFIERS[5],
404 true));
405 result = ht.get(get);
406 kvListExp = new ArrayList<Cell>();
407 kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[3], 1, VALUE));
408 kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[4], 1, VALUE));
409 kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[5], 1, VALUE));
410 verifyResult(result, kvListExp, toLog, "Testing offset with CRF");
411
412
413
414 for(int j=2; j > 0; j--) {
415 put = new Put(ROW);
416 for (int i=0; i < 10; i++) {
417 KeyValue kv = new KeyValue(ROW, FAMILIES[j], QUALIFIERS[i], 1, VALUE);
418 put.add(kv);
419 }
420 ht.put(put);
421 }
422
423 get = new Get(ROW);
424 get.setRowOffsetPerColumnFamily(4);
425 get.setMaxResultsPerColumnFamily(2);
426 get.addFamily(FAMILIES[1]);
427 get.addFamily(FAMILIES[2]);
428 result = ht.get(get);
429 kvListExp = new ArrayList<Cell>();
430
431 kvListExp.add(new KeyValue(ROW, FAMILIES[1], QUALIFIERS[4], 1, VALUE));
432 kvListExp.add(new KeyValue(ROW, FAMILIES[1], QUALIFIERS[5], 1, VALUE));
433 kvListExp.add(new KeyValue(ROW, FAMILIES[2], QUALIFIERS[4], 1, VALUE));
434 kvListExp.add(new KeyValue(ROW, FAMILIES[2], QUALIFIERS[5], 1, VALUE));
435 verifyResult(result, kvListExp, toLog,
436 "Testing offset + multiple CFs + maxResults");
437 }
438
439
440
441
442
443
444
445 @Test
446 public void testScanOnReopenedRegion() throws Exception {
447 byte [] TABLE = Bytes.toBytes("testScanOnReopenedRegion");
448 byte [][] QUALIFIERS = HTestConst.makeNAscii(QUALIFIER, 2);
449
450 HTable ht = TEST_UTIL.createTable(TABLE, FAMILY);
451
452 Put put;
453 Scan scan;
454 Result result;
455 ResultScanner scanner;
456 boolean toLog = false;
457 List<Cell> kvListExp;
458
459
460 put = new Put(ROW);
461 for (int i=0; i < QUALIFIERS.length; i++) {
462 KeyValue kv = new KeyValue(ROW, FAMILY, QUALIFIERS[i], i, VALUE);
463 put.add(kv);
464 }
465 ht.put(put);
466
467 scan = new Scan(ROW);
468 scanner = ht.getScanner(scan);
469
470 HRegionLocation loc = ht.getRegionLocation(ROW);
471 HRegionInfo hri = loc.getRegionInfo();
472 MiniHBaseCluster cluster = TEST_UTIL.getMiniHBaseCluster();
473 byte[] regionName = hri.getRegionName();
474 int i = cluster.getServerWith(regionName);
475 HRegionServer rs = cluster.getRegionServer(i);
476 ProtobufUtil.closeRegion(rs, rs.getServerName(), regionName, false);
477 long startTime = EnvironmentEdgeManager.currentTimeMillis();
478 long timeOut = 300000;
479 while (true) {
480 if (rs.getOnlineRegion(regionName) == null) {
481 break;
482 }
483 assertTrue("Timed out in closing the testing region",
484 EnvironmentEdgeManager.currentTimeMillis() < startTime + timeOut);
485 Thread.sleep(500);
486 }
487
488
489 ZooKeeperWatcher zkw = TEST_UTIL.getZooKeeperWatcher();
490 try {
491 HMaster master = cluster.getMaster();
492 RegionStates states = master.getAssignmentManager().getRegionStates();
493 states.regionOffline(hri);
494 states.updateRegionState(hri, State.OPENING);
495 if (ConfigUtil.useZKForAssignment(TEST_UTIL.getConfiguration())) {
496 ZKAssign.createNodeOffline(zkw, hri, loc.getServerName());
497 }
498 ProtobufUtil.openRegion(rs, rs.getServerName(), hri);
499 startTime = EnvironmentEdgeManager.currentTimeMillis();
500 while (true) {
501 if (rs.getOnlineRegion(regionName) != null) {
502 break;
503 }
504 assertTrue("Timed out in open the testing region",
505 EnvironmentEdgeManager.currentTimeMillis() < startTime + timeOut);
506 Thread.sleep(500);
507 }
508 } finally {
509 ZKAssign.deleteNodeFailSilent(zkw, hri);
510 }
511
512
513 kvListExp = new ArrayList<Cell>();
514 kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[0], 0, VALUE));
515 kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[1], 1, VALUE));
516 result = scanner.next();
517 verifyResult(result, kvListExp, toLog, "Testing scan on re-opened region");
518 }
519
520 static void verifyResult(Result result, List<Cell> expKvList, boolean toLog,
521 String msg) {
522
523 LOG.info(msg);
524 LOG.info("Expected count: " + expKvList.size());
525 LOG.info("Actual count: " + result.size());
526 if (expKvList.size() == 0)
527 return;
528
529 int i = 0;
530 for (Cell kv : result.rawCells()) {
531 if (i >= expKvList.size()) {
532 break;
533 }
534
535 Cell kvExp = expKvList.get(i++);
536 if (toLog) {
537 LOG.info("get kv is: " + kv.toString());
538 LOG.info("exp kv is: " + kvExp.toString());
539 }
540 assertTrue("Not equal", kvExp.equals(kv));
541 }
542
543 assertEquals(expKvList.size(), result.size());
544 }
545
546
547 }