View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one or more
3    * contributor license agreements. See the NOTICE file distributed with this
4    * work for additional information regarding copyright ownership. The ASF
5    * licenses this file to you under the Apache License, Version 2.0 (the
6    * "License"); you may not use this file except in compliance with the License.
7    * You may obtain a copy of the License at
8    *
9    * http://www.apache.org/licenses/LICENSE-2.0
10   *
11   * Unless required by applicable law or agreed to in writing, software
12   * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
13   * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
14   * License for the specific language governing permissions and limitations
15   * under the License.
16   */
17  package org.apache.hadoop.hbase.client;
18  
19  import static org.junit.Assert.assertEquals;
20  import static org.junit.Assert.assertTrue;
21  
22  import java.util.ArrayList;
23  import java.util.List;
24  
25  import org.apache.commons.logging.Log;
26  import org.apache.commons.logging.LogFactory;
27  import org.apache.hadoop.hbase.Cell;
28  import org.apache.hadoop.hbase.HBaseTestingUtility;
29  import org.apache.hadoop.hbase.HRegionInfo;
30  import org.apache.hadoop.hbase.HRegionLocation;
31  import org.apache.hadoop.hbase.HTestConst;
32  import org.apache.hadoop.hbase.KeyValue;
33  import org.apache.hadoop.hbase.testclassification.MediumTests;
34  import org.apache.hadoop.hbase.MiniHBaseCluster;
35  import org.apache.hadoop.hbase.filter.ColumnPrefixFilter;
36  import org.apache.hadoop.hbase.filter.ColumnRangeFilter;
37  import org.apache.hadoop.hbase.master.HMaster;
38  import org.apache.hadoop.hbase.master.RegionState.State;
39  import org.apache.hadoop.hbase.master.RegionStates;
40  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
41  import org.apache.hadoop.hbase.regionserver.HRegionServer;
42  import org.apache.hadoop.hbase.util.Bytes;
43  import org.apache.hadoop.hbase.util.ConfigUtil;
44  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
45  import org.apache.hadoop.hbase.zookeeper.ZKAssign;
46  import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
47  import org.junit.After;
48  import org.junit.AfterClass;
49  import org.junit.Before;
50  import org.junit.BeforeClass;
51  import org.junit.Test;
52  import org.junit.experimental.categories.Category;
53  
54  /**
55   * A client-side test, mostly testing scanners with various parameters.
56   */
57  @Category(MediumTests.class)
58  public class TestScannersFromClientSide {
59    private static final Log LOG = LogFactory.getLog(TestScannersFromClientSide.class);
60  
61    private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
62    private static byte [] ROW = Bytes.toBytes("testRow");
63    private static byte [] FAMILY = Bytes.toBytes("testFamily");
64    private static byte [] QUALIFIER = Bytes.toBytes("testQualifier");
65    private static byte [] VALUE = Bytes.toBytes("testValue");
66  
67    /**
68     * @throws java.lang.Exception
69     */
70    @BeforeClass
71    public static void setUpBeforeClass() throws Exception {
72      TEST_UTIL.startMiniCluster(3);
73    }
74  
75    /**
76     * @throws java.lang.Exception
77     */
78    @AfterClass
79    public static void tearDownAfterClass() throws Exception {
80      TEST_UTIL.shutdownMiniCluster();
81    }
82  
83    /**
84     * @throws java.lang.Exception
85     */
86    @Before
87    public void setUp() throws Exception {
88      // Nothing to do.
89    }
90  
91    /**
92     * @throws java.lang.Exception
93     */
94    @After
95    public void tearDown() throws Exception {
96      // Nothing to do.
97    }
98  
99    /**
100    * Test from client side for batch of scan
101    *
102    * @throws Exception
103    */
104   @Test
105   public void testScanBatch() throws Exception {
106     byte [] TABLE = Bytes.toBytes("testScanBatch");
107     byte [][] QUALIFIERS = HTestConst.makeNAscii(QUALIFIER, 8);
108 
109     HTable ht = TEST_UTIL.createTable(TABLE, FAMILY);
110 
111     Put put;
112     Scan scan;
113     Delete delete;
114     Result result;
115     ResultScanner scanner;
116     boolean toLog = true;
117     List<Cell> kvListExp;
118 
119     // table: row, family, c0:0, c1:1, ... , c7:7
120     put = new Put(ROW);
121     for (int i=0; i < QUALIFIERS.length; i++) {
122       KeyValue kv = new KeyValue(ROW, FAMILY, QUALIFIERS[i], i, VALUE);
123       put.add(kv);
124     }
125     ht.put(put);
126 
127     // table: row, family, c0:0, c1:1, ..., c6:2, c6:6 , c7:7
128     put = new Put(ROW);
129     KeyValue kv = new KeyValue(ROW, FAMILY, QUALIFIERS[6], 2, VALUE);
130     put.add(kv);
131     ht.put(put);
132 
133     // delete upto ts: 3
134     delete = new Delete(ROW);
135     delete.deleteFamily(FAMILY, 3);
136     ht.delete(delete);
137 
138     // without batch
139     scan = new Scan(ROW);
140     scan.setMaxVersions();
141     scanner = ht.getScanner(scan);
142 
143     // c4:4, c5:5, c6:6, c7:7
144     kvListExp = new ArrayList<Cell>();
145     kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[4], 4, VALUE));
146     kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[5], 5, VALUE));
147     kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[6], 6, VALUE));
148     kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[7], 7, VALUE));
149     result = scanner.next();
150     verifyResult(result, kvListExp, toLog, "Testing first batch of scan");
151 
152     // with batch
153     scan = new Scan(ROW);
154     scan.setMaxVersions();
155     scan.setBatch(2);
156     scanner = ht.getScanner(scan);
157 
158     // First batch: c4:4, c5:5
159     kvListExp = new ArrayList<Cell>();
160     kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[4], 4, VALUE));
161     kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[5], 5, VALUE));
162     result = scanner.next();
163     verifyResult(result, kvListExp, toLog, "Testing first batch of scan");
164 
165     // Second batch: c6:6, c7:7
166     kvListExp = new ArrayList<Cell>();
167     kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[6], 6, VALUE));
168     kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[7], 7, VALUE));
169     result = scanner.next();
170     verifyResult(result, kvListExp, toLog, "Testing second batch of scan");
171 
172   }
173 
174   /**
175    * Test from client side for get with maxResultPerCF set
176    *
177    * @throws Exception
178    */
179   @Test
180   public void testGetMaxResults() throws Exception {
181     byte [] TABLE = Bytes.toBytes("testGetMaxResults");
182     byte [][] FAMILIES = HTestConst.makeNAscii(FAMILY, 3);
183     byte [][] QUALIFIERS = HTestConst.makeNAscii(QUALIFIER, 20);
184 
185     HTable ht = TEST_UTIL.createTable(TABLE, FAMILIES);
186 
187     Get get;
188     Put put;
189     Result result;
190     boolean toLog = true;
191     List<Cell> kvListExp;
192 
193     kvListExp = new ArrayList<Cell>();
194     // Insert one CF for row[0]
195     put = new Put(ROW);
196     for (int i=0; i < 10; i++) {
197       KeyValue kv = new KeyValue(ROW, FAMILIES[0], QUALIFIERS[i], 1, VALUE);
198       put.add(kv);
199       kvListExp.add(kv);
200     }
201     ht.put(put);
202 
203     get = new Get(ROW);
204     result = ht.get(get);
205     verifyResult(result, kvListExp, toLog, "Testing without setting maxResults");
206 
207     get = new Get(ROW);
208     get.setMaxResultsPerColumnFamily(2);
209     result = ht.get(get);
210     kvListExp = new ArrayList<Cell>();
211     kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[0], 1, VALUE));
212     kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[1], 1, VALUE));
213     verifyResult(result, kvListExp, toLog, "Testing basic setMaxResults");
214 
215     // Filters: ColumnRangeFilter
216     get = new Get(ROW);
217     get.setMaxResultsPerColumnFamily(5);
218     get.setFilter(new ColumnRangeFilter(QUALIFIERS[2], true, QUALIFIERS[5],
219                                         true));
220     result = ht.get(get);
221     kvListExp = new ArrayList<Cell>();
222     kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[2], 1, VALUE));
223     kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[3], 1, VALUE));
224     kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[4], 1, VALUE));
225     kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[5], 1, VALUE));
226     verifyResult(result, kvListExp, toLog, "Testing single CF with CRF");
227 
228     // Insert two more CF for row[0]
229     // 20 columns for CF2, 10 columns for CF1
230     put = new Put(ROW);
231     for (int i=0; i < QUALIFIERS.length; i++) {
232       KeyValue kv = new KeyValue(ROW, FAMILIES[2], QUALIFIERS[i], 1, VALUE);
233       put.add(kv);
234     }
235     ht.put(put);
236 
237     put = new Put(ROW);
238     for (int i=0; i < 10; i++) {
239       KeyValue kv = new KeyValue(ROW, FAMILIES[1], QUALIFIERS[i], 1, VALUE);
240       put.add(kv);
241     }
242     ht.put(put);
243 
244     get = new Get(ROW);
245     get.setMaxResultsPerColumnFamily(12);
246     get.addFamily(FAMILIES[1]);
247     get.addFamily(FAMILIES[2]);
248     result = ht.get(get);
249     kvListExp = new ArrayList<Cell>();
250     //Exp: CF1:q0, ..., q9, CF2: q0, q1, q10, q11, ..., q19
251     for (int i=0; i < 10; i++) {
252       kvListExp.add(new KeyValue(ROW, FAMILIES[1], QUALIFIERS[i], 1, VALUE));
253     }
254     for (int i=0; i < 2; i++) {
255         kvListExp.add(new KeyValue(ROW, FAMILIES[2], QUALIFIERS[i], 1, VALUE));
256       }
257     for (int i=10; i < 20; i++) {
258       kvListExp.add(new KeyValue(ROW, FAMILIES[2], QUALIFIERS[i], 1, VALUE));
259     }
260     verifyResult(result, kvListExp, toLog, "Testing multiple CFs");
261 
262     // Filters: ColumnRangeFilter and ColumnPrefixFilter
263     get = new Get(ROW);
264     get.setMaxResultsPerColumnFamily(3);
265     get.setFilter(new ColumnRangeFilter(QUALIFIERS[2], true, null, true));
266     result = ht.get(get);
267     kvListExp = new ArrayList<Cell>();
268     for (int i=2; i < 5; i++) {
269       kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[i], 1, VALUE));
270     }
271     for (int i=2; i < 5; i++) {
272       kvListExp.add(new KeyValue(ROW, FAMILIES[1], QUALIFIERS[i], 1, VALUE));
273     }
274     for (int i=2; i < 5; i++) {
275       kvListExp.add(new KeyValue(ROW, FAMILIES[2], QUALIFIERS[i], 1, VALUE));
276     }
277     verifyResult(result, kvListExp, toLog, "Testing multiple CFs + CRF");
278 
279     get = new Get(ROW);
280     get.setMaxResultsPerColumnFamily(7);
281     get.setFilter(new ColumnPrefixFilter(QUALIFIERS[1]));
282     result = ht.get(get);
283     kvListExp = new ArrayList<Cell>();
284     kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[1], 1, VALUE));
285     kvListExp.add(new KeyValue(ROW, FAMILIES[1], QUALIFIERS[1], 1, VALUE));
286     kvListExp.add(new KeyValue(ROW, FAMILIES[2], QUALIFIERS[1], 1, VALUE));
287     for (int i=10; i < 16; i++) {
288       kvListExp.add(new KeyValue(ROW, FAMILIES[2], QUALIFIERS[i], 1, VALUE));
289     }
290     verifyResult(result, kvListExp, toLog, "Testing multiple CFs + PFF");
291 
292   }
293 
294   /**
295    * Test from client side for scan with maxResultPerCF set
296    *
297    * @throws Exception
298    */
299   @Test
300   public void testScanMaxResults() throws Exception {
301     byte [] TABLE = Bytes.toBytes("testScanLimit");
302     byte [][] ROWS = HTestConst.makeNAscii(ROW, 2);
303     byte [][] FAMILIES = HTestConst.makeNAscii(FAMILY, 3);
304     byte [][] QUALIFIERS = HTestConst.makeNAscii(QUALIFIER, 10);
305 
306     HTable ht = TEST_UTIL.createTable(TABLE, FAMILIES);
307 
308     Put put;
309     Scan scan;
310     Result result;
311     boolean toLog = true;
312     List<Cell> kvListExp, kvListScan;
313 
314     kvListExp = new ArrayList<Cell>();
315 
316     for (int r=0; r < ROWS.length; r++) {
317       put = new Put(ROWS[r]);
318       for (int c=0; c < FAMILIES.length; c++) {
319         for (int q=0; q < QUALIFIERS.length; q++) {
320           KeyValue kv = new KeyValue(ROWS[r], FAMILIES[c], QUALIFIERS[q], 1, VALUE);
321           put.add(kv);
322           if (q < 4) {
323             kvListExp.add(kv);
324           }
325         }
326       }
327       ht.put(put);
328     }
329 
330     scan = new Scan();
331     scan.setMaxResultsPerColumnFamily(4);
332     ResultScanner scanner = ht.getScanner(scan);
333     kvListScan = new ArrayList<Cell>();
334     while ((result = scanner.next()) != null) {
335       for (Cell kv : result.listCells()) {
336         kvListScan.add(kv);
337       }
338     }
339     result = Result.create(kvListScan);
340     verifyResult(result, kvListExp, toLog, "Testing scan with maxResults");
341 
342   }
343 
344   /**
345    * Test from client side for get with rowOffset
346    *
347    * @throws Exception
348    */
349   @Test
350   public void testGetRowOffset() throws Exception {
351     byte [] TABLE = Bytes.toBytes("testGetRowOffset");
352     byte [][] FAMILIES = HTestConst.makeNAscii(FAMILY, 3);
353     byte [][] QUALIFIERS = HTestConst.makeNAscii(QUALIFIER, 20);
354 
355     HTable ht = TEST_UTIL.createTable(TABLE, FAMILIES);
356 
357     Get get;
358     Put put;
359     Result result;
360     boolean toLog = true;
361     List<Cell> kvListExp;
362 
363     // Insert one CF for row
364     kvListExp = new ArrayList<Cell>();
365     put = new Put(ROW);
366     for (int i=0; i < 10; i++) {
367       KeyValue kv = new KeyValue(ROW, FAMILIES[0], QUALIFIERS[i], 1, VALUE);
368       put.add(kv);
369       // skipping first two kvs
370       if (i < 2) continue;
371       kvListExp.add(kv);
372     }
373     ht.put(put);
374 
375     //setting offset to 2
376     get = new Get(ROW);
377     get.setRowOffsetPerColumnFamily(2);
378     result = ht.get(get);
379     verifyResult(result, kvListExp, toLog, "Testing basic setRowOffset");
380 
381     //setting offset to 20
382     get = new Get(ROW);
383     get.setRowOffsetPerColumnFamily(20);
384     result = ht.get(get);
385     kvListExp = new ArrayList<Cell>();
386     verifyResult(result, kvListExp, toLog, "Testing offset > #kvs");
387 
388     //offset + maxResultPerCF
389     get = new Get(ROW);
390     get.setRowOffsetPerColumnFamily(4);
391     get.setMaxResultsPerColumnFamily(5);
392     result = ht.get(get);
393     kvListExp = new ArrayList<Cell>();
394     for (int i=4; i < 9; i++) {
395       kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[i], 1, VALUE));
396     }
397     verifyResult(result, kvListExp, toLog,
398       "Testing offset + setMaxResultsPerCF");
399 
400     // Filters: ColumnRangeFilter
401     get = new Get(ROW);
402     get.setRowOffsetPerColumnFamily(1);
403     get.setFilter(new ColumnRangeFilter(QUALIFIERS[2], true, QUALIFIERS[5],
404                                         true));
405     result = ht.get(get);
406     kvListExp = new ArrayList<Cell>();
407     kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[3], 1, VALUE));
408     kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[4], 1, VALUE));
409     kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[5], 1, VALUE));
410     verifyResult(result, kvListExp, toLog, "Testing offset with CRF");
411 
412     // Insert into two more CFs for row
413     // 10 columns for CF2, 10 columns for CF1
414     for(int j=2; j > 0; j--) {
415       put = new Put(ROW);
416       for (int i=0; i < 10; i++) {
417         KeyValue kv = new KeyValue(ROW, FAMILIES[j], QUALIFIERS[i], 1, VALUE);
418         put.add(kv);
419       }
420       ht.put(put);
421     }
422 
423     get = new Get(ROW);
424     get.setRowOffsetPerColumnFamily(4);
425     get.setMaxResultsPerColumnFamily(2);
426     get.addFamily(FAMILIES[1]);
427     get.addFamily(FAMILIES[2]);
428     result = ht.get(get);
429     kvListExp = new ArrayList<Cell>();
430     //Exp: CF1:q4, q5, CF2: q4, q5
431     kvListExp.add(new KeyValue(ROW, FAMILIES[1], QUALIFIERS[4], 1, VALUE));
432     kvListExp.add(new KeyValue(ROW, FAMILIES[1], QUALIFIERS[5], 1, VALUE));
433     kvListExp.add(new KeyValue(ROW, FAMILIES[2], QUALIFIERS[4], 1, VALUE));
434     kvListExp.add(new KeyValue(ROW, FAMILIES[2], QUALIFIERS[5], 1, VALUE));
435     verifyResult(result, kvListExp, toLog,
436        "Testing offset + multiple CFs + maxResults");
437   }
438 
439   /**
440    * Test from client side for scan while the region is reopened
441    * on the same region server.
442    *
443    * @throws Exception
444    */
445   @Test
446   public void testScanOnReopenedRegion() throws Exception {
447     byte [] TABLE = Bytes.toBytes("testScanOnReopenedRegion");
448     byte [][] QUALIFIERS = HTestConst.makeNAscii(QUALIFIER, 2);
449 
450     HTable ht = TEST_UTIL.createTable(TABLE, FAMILY);
451 
452     Put put;
453     Scan scan;
454     Result result;
455     ResultScanner scanner;
456     boolean toLog = false;
457     List<Cell> kvListExp;
458 
459     // table: row, family, c0:0, c1:1
460     put = new Put(ROW);
461     for (int i=0; i < QUALIFIERS.length; i++) {
462       KeyValue kv = new KeyValue(ROW, FAMILY, QUALIFIERS[i], i, VALUE);
463       put.add(kv);
464     }
465     ht.put(put);
466 
467     scan = new Scan(ROW);
468     scanner = ht.getScanner(scan);
469 
470     HRegionLocation loc = ht.getRegionLocation(ROW);
471     HRegionInfo hri = loc.getRegionInfo();
472     MiniHBaseCluster cluster = TEST_UTIL.getMiniHBaseCluster();
473     byte[] regionName = hri.getRegionName();
474     int i = cluster.getServerWith(regionName);
475     HRegionServer rs = cluster.getRegionServer(i);
476     ProtobufUtil.closeRegion(rs, rs.getServerName(), regionName, false);
477     long startTime = EnvironmentEdgeManager.currentTimeMillis();
478     long timeOut = 300000;
479     while (true) {
480       if (rs.getOnlineRegion(regionName) == null) {
481         break;
482       }
483       assertTrue("Timed out in closing the testing region",
484         EnvironmentEdgeManager.currentTimeMillis() < startTime + timeOut);
485       Thread.sleep(500);
486     }
487 
488     // Now open the region again.
489     ZooKeeperWatcher zkw = TEST_UTIL.getZooKeeperWatcher();
490     try {
491       HMaster master = cluster.getMaster();
492       RegionStates states = master.getAssignmentManager().getRegionStates();
493       states.regionOffline(hri);
494       states.updateRegionState(hri, State.OPENING);
495       if (ConfigUtil.useZKForAssignment(TEST_UTIL.getConfiguration())) {
496         ZKAssign.createNodeOffline(zkw, hri, loc.getServerName());
497       }
498       ProtobufUtil.openRegion(rs, rs.getServerName(), hri);
499       startTime = EnvironmentEdgeManager.currentTimeMillis();
500       while (true) {
501         if (rs.getOnlineRegion(regionName) != null) {
502           break;
503         }
504         assertTrue("Timed out in open the testing region",
505           EnvironmentEdgeManager.currentTimeMillis() < startTime + timeOut);
506         Thread.sleep(500);
507       }
508     } finally {
509       ZKAssign.deleteNodeFailSilent(zkw, hri);
510     }
511 
512     // c0:0, c1:1
513     kvListExp = new ArrayList<Cell>();
514     kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[0], 0, VALUE));
515     kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[1], 1, VALUE));
516     result = scanner.next();
517     verifyResult(result, kvListExp, toLog, "Testing scan on re-opened region");
518   }
519 
520   static void verifyResult(Result result, List<Cell> expKvList, boolean toLog,
521       String msg) {
522 
523     LOG.info(msg);
524     LOG.info("Expected count: " + expKvList.size());
525     LOG.info("Actual count: " + result.size());
526     if (expKvList.size() == 0)
527       return;
528 
529     int i = 0;
530     for (Cell kv : result.rawCells()) {
531       if (i >= expKvList.size()) {
532         break;  // we will check the size later
533       }
534 
535       Cell kvExp = expKvList.get(i++);
536       if (toLog) {
537         LOG.info("get kv is: " + kv.toString());
538         LOG.info("exp kv is: " + kvExp.toString());
539       }
540       assertTrue("Not equal", kvExp.equals(kv));
541     }
542 
543     assertEquals(expKvList.size(), result.size());
544   }
545 
546 
547 }