View Javadoc

1   /*
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  
20  package org.apache.hadoop.hbase.regionserver;
21  
22  import static org.apache.hadoop.hbase.regionserver.ScanQueryMatcher.MatchCode.INCLUDE;
23  import static org.apache.hadoop.hbase.regionserver.ScanQueryMatcher.MatchCode.SKIP;
24  
25  import java.io.IOException;
26  import java.util.ArrayList;
27  import java.util.List;
28  import java.util.NavigableSet;
29  
30  import org.apache.hadoop.hbase.HBaseTestCase;
31  import org.apache.hadoop.hbase.HConstants;
32  import org.apache.hadoop.hbase.KeepDeletedCells;
33  import org.apache.hadoop.hbase.KeyValue;
34  import org.apache.hadoop.hbase.KeyValue.KVComparator;
35  import org.apache.hadoop.hbase.KeyValue.Type;
36  import org.apache.hadoop.hbase.client.Get;
37  import org.apache.hadoop.hbase.client.Scan;
38  import org.apache.hadoop.hbase.regionserver.ScanQueryMatcher.MatchCode;
39  import org.apache.hadoop.hbase.testclassification.SmallTests;
40  import org.apache.hadoop.hbase.util.Bytes;
41  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
42  import org.junit.experimental.categories.Category;
43  
44  @Category(SmallTests.class)
45  public class TestQueryMatcher extends HBaseTestCase {
46    private static final boolean PRINT = false;
47  
48    private byte[] row1;
49    private byte[] row2;
50    private byte[] row3;
51    private byte[] fam1;
52    private byte[] fam2;
53    private byte[] col1;
54    private byte[] col2;
55    private byte[] col3;
56    private byte[] col4;
57    private byte[] col5;
58  
59    private byte[] data;
60  
61    private Get get;
62  
63    long ttl = Long.MAX_VALUE;
64    KVComparator rowComparator;
65    private Scan scan;
66  
67    public void setUp() throws Exception {
68      super.setUp();
69      row1 = Bytes.toBytes("row1");
70      row2 = Bytes.toBytes("row2");
71      row3 = Bytes.toBytes("row3");
72      fam1 = Bytes.toBytes("fam1");
73      fam2 = Bytes.toBytes("fam2");
74      col1 = Bytes.toBytes("col1");
75      col2 = Bytes.toBytes("col2");
76      col3 = Bytes.toBytes("col3");
77      col4 = Bytes.toBytes("col4");
78      col5 = Bytes.toBytes("col5");
79  
80      data = Bytes.toBytes("data");
81  
82      //Create Get
83      get = new Get(row1);
84      get.addFamily(fam1);
85      get.addColumn(fam2, col2);
86      get.addColumn(fam2, col4);
87      get.addColumn(fam2, col5);
88      this.scan = new Scan(get);
89  
90      rowComparator = KeyValue.COMPARATOR;
91  
92    }
93  
94    private void _testMatch_ExplicitColumns(Scan scan, List<MatchCode> expected) throws IOException {
95      long now = EnvironmentEdgeManager.currentTimeMillis();
96      // 2,4,5
97      ScanQueryMatcher qm = new ScanQueryMatcher(scan, new ScanInfo(fam2,
98          0, 1, ttl, KeepDeletedCells.FALSE, 0, rowComparator), get.getFamilyMap().get(fam2),
99          now - ttl, now);
100 
101     List<KeyValue> memstore = new ArrayList<KeyValue>();
102     memstore.add(new KeyValue(row1, fam2, col1, 1, data));
103     memstore.add(new KeyValue(row1, fam2, col2, 1, data));
104     memstore.add(new KeyValue(row1, fam2, col3, 1, data));
105     memstore.add(new KeyValue(row1, fam2, col4, 1, data));
106     memstore.add(new KeyValue(row1, fam2, col5, 1, data));
107 
108     memstore.add(new KeyValue(row2, fam1, col1, data));
109 
110     List<ScanQueryMatcher.MatchCode> actual = new ArrayList<ScanQueryMatcher.MatchCode>();
111     KeyValue k = memstore.get(0);
112     qm.setRow(k.getBuffer(), k.getRowOffset(), k.getRowLength());
113 
114     for (KeyValue kv : memstore){
115       actual.add(qm.match(kv));
116     }
117 
118     assertEquals(expected.size(), actual.size());
119     for(int i=0; i< expected.size(); i++){
120       assertEquals(expected.get(i), actual.get(i));
121       if(PRINT){
122         System.out.println("expected "+expected.get(i)+
123             ", actual " +actual.get(i));
124       }
125     }
126   }
127 
128   public void testMatch_ExplicitColumns()
129   throws IOException {
130     //Moving up from the Tracker by using Gets and List<KeyValue> instead
131     //of just byte []
132 
133     //Expected result
134     List<MatchCode> expected = new ArrayList<ScanQueryMatcher.MatchCode>();
135     expected.add(ScanQueryMatcher.MatchCode.SEEK_NEXT_COL);
136     expected.add(ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL);
137     expected.add(ScanQueryMatcher.MatchCode.SEEK_NEXT_COL);
138     expected.add(ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL);
139     expected.add(ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_ROW);
140     expected.add(ScanQueryMatcher.MatchCode.DONE);
141 
142     _testMatch_ExplicitColumns(scan, expected);
143   }
144 
145   public void testMatch_ExplicitColumnsWithLookAhead()
146   throws IOException {
147     //Moving up from the Tracker by using Gets and List<KeyValue> instead
148     //of just byte []
149 
150     //Expected result
151     List<MatchCode> expected = new ArrayList<ScanQueryMatcher.MatchCode>();
152     expected.add(ScanQueryMatcher.MatchCode.SKIP);
153     expected.add(ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL);
154     expected.add(ScanQueryMatcher.MatchCode.SKIP);
155     expected.add(ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL);
156     expected.add(ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_ROW);
157     expected.add(ScanQueryMatcher.MatchCode.DONE);
158 
159     Scan s = new Scan(scan);
160     s.setAttribute(Scan.HINT_LOOKAHEAD, Bytes.toBytes(2));
161     _testMatch_ExplicitColumns(s, expected);
162   }
163 
164 
165   public void testMatch_Wildcard()
166   throws IOException {
167     //Moving up from the Tracker by using Gets and List<KeyValue> instead
168     //of just byte []
169 
170     //Expected result
171     List<MatchCode> expected = new ArrayList<ScanQueryMatcher.MatchCode>();
172     expected.add(ScanQueryMatcher.MatchCode.INCLUDE);
173     expected.add(ScanQueryMatcher.MatchCode.INCLUDE);
174     expected.add(ScanQueryMatcher.MatchCode.INCLUDE);
175     expected.add(ScanQueryMatcher.MatchCode.INCLUDE);
176     expected.add(ScanQueryMatcher.MatchCode.INCLUDE);
177     expected.add(ScanQueryMatcher.MatchCode.DONE);
178 
179     long now = EnvironmentEdgeManager.currentTimeMillis();
180     ScanQueryMatcher qm = new ScanQueryMatcher(scan, new ScanInfo(fam2,
181         0, 1, ttl, KeepDeletedCells.FALSE, 0, rowComparator), null,
182         now - ttl, now);
183 
184     List<KeyValue> memstore = new ArrayList<KeyValue>();
185     memstore.add(new KeyValue(row1, fam2, col1, 1, data));
186     memstore.add(new KeyValue(row1, fam2, col2, 1, data));
187     memstore.add(new KeyValue(row1, fam2, col3, 1, data));
188     memstore.add(new KeyValue(row1, fam2, col4, 1, data));
189     memstore.add(new KeyValue(row1, fam2, col5, 1, data));
190     memstore.add(new KeyValue(row2, fam1, col1, 1, data));
191 
192     List<ScanQueryMatcher.MatchCode> actual = new ArrayList<ScanQueryMatcher.MatchCode>();
193 
194     KeyValue k = memstore.get(0);
195     qm.setRow(k.getBuffer(), k.getRowOffset(), k.getRowLength());
196 
197     for(KeyValue kv : memstore) {
198       actual.add(qm.match(kv));
199     }
200 
201     assertEquals(expected.size(), actual.size());
202     for(int i=0; i< expected.size(); i++){
203       assertEquals(expected.get(i), actual.get(i));
204       if(PRINT){
205         System.out.println("expected "+expected.get(i)+
206             ", actual " +actual.get(i));
207       }
208     }
209   }
210 
211 
212   /**
213    * Verify that {@link ScanQueryMatcher} only skips expired KeyValue
214    * instances and does not exit early from the row (skipping
215    * later non-expired KeyValues).  This version mimics a Get with
216    * explicitly specified column qualifiers.
217    *
218    * @throws IOException
219    */
220   public void testMatch_ExpiredExplicit()
221   throws IOException {
222 
223     long testTTL = 1000;
224     MatchCode [] expected = new MatchCode[] {
225         ScanQueryMatcher.MatchCode.SEEK_NEXT_COL,
226         ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL,
227         ScanQueryMatcher.MatchCode.SEEK_NEXT_COL,
228         ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL,
229         ScanQueryMatcher.MatchCode.SEEK_NEXT_ROW,
230         ScanQueryMatcher.MatchCode.DONE
231     };
232 
233     long now = EnvironmentEdgeManager.currentTimeMillis();
234     ScanQueryMatcher qm = new ScanQueryMatcher(scan, new ScanInfo(fam2,
235         0, 1, testTTL, KeepDeletedCells.FALSE, 0, rowComparator), get.getFamilyMap().get(fam2),
236         now - testTTL, now);
237 
238     KeyValue [] kvs = new KeyValue[] {
239         new KeyValue(row1, fam2, col1, now-100, data),
240         new KeyValue(row1, fam2, col2, now-50, data),
241         new KeyValue(row1, fam2, col3, now-5000, data),
242         new KeyValue(row1, fam2, col4, now-500, data),
243         new KeyValue(row1, fam2, col5, now-10000, data),
244         new KeyValue(row2, fam1, col1, now-10, data)
245     };
246 
247     KeyValue k = kvs[0];
248     qm.setRow(k.getBuffer(), k.getRowOffset(), k.getRowLength());
249 
250     List<MatchCode> actual = new ArrayList<MatchCode>(kvs.length);
251     for (KeyValue kv : kvs) {
252       actual.add( qm.match(kv) );
253     }
254 
255     assertEquals(expected.length, actual.size());
256     for (int i=0; i<expected.length; i++) {
257       if(PRINT){
258         System.out.println("expected "+expected[i]+
259             ", actual " +actual.get(i));
260       }
261       assertEquals(expected[i], actual.get(i));
262     }
263   }
264 
265 
266   /**
267    * Verify that {@link ScanQueryMatcher} only skips expired KeyValue
268    * instances and does not exit early from the row (skipping
269    * later non-expired KeyValues).  This version mimics a Get with
270    * wildcard-inferred column qualifiers.
271    *
272    * @throws IOException
273    */
274   public void testMatch_ExpiredWildcard()
275   throws IOException {
276 
277     long testTTL = 1000;
278     MatchCode [] expected = new MatchCode[] {
279         ScanQueryMatcher.MatchCode.INCLUDE,
280         ScanQueryMatcher.MatchCode.INCLUDE,
281         ScanQueryMatcher.MatchCode.SEEK_NEXT_COL,
282         ScanQueryMatcher.MatchCode.INCLUDE,
283         ScanQueryMatcher.MatchCode.SEEK_NEXT_COL,
284         ScanQueryMatcher.MatchCode.DONE
285     };
286 
287     long now = EnvironmentEdgeManager.currentTimeMillis();
288     ScanQueryMatcher qm = new ScanQueryMatcher(scan, new ScanInfo(fam2,
289         0, 1, testTTL, KeepDeletedCells.FALSE, 0, rowComparator), null,
290         now - testTTL, now);
291 
292     KeyValue [] kvs = new KeyValue[] {
293         new KeyValue(row1, fam2, col1, now-100, data),
294         new KeyValue(row1, fam2, col2, now-50, data),
295         new KeyValue(row1, fam2, col3, now-5000, data),
296         new KeyValue(row1, fam2, col4, now-500, data),
297         new KeyValue(row1, fam2, col5, now-10000, data),
298         new KeyValue(row2, fam1, col1, now-10, data)
299     };
300     KeyValue k = kvs[0];
301     qm.setRow(k.getBuffer(), k.getRowOffset(), k.getRowLength());
302 
303     List<ScanQueryMatcher.MatchCode> actual =
304         new ArrayList<ScanQueryMatcher.MatchCode>(kvs.length);
305     for (KeyValue kv : kvs) {
306       actual.add( qm.match(kv) );
307     }
308 
309     assertEquals(expected.length, actual.size());
310     for (int i=0; i<expected.length; i++) {
311       if(PRINT){
312         System.out.println("expected "+expected[i]+
313             ", actual " +actual.get(i));
314       }
315       assertEquals(expected[i], actual.get(i));
316     }
317   }
318 
319   public void testMatch_PartialRangeDropDeletes() throws Exception {
320     // Some ranges.
321     testDropDeletes(
322         row2, row3, new byte[][] { row1, row2, row2, row3 }, INCLUDE, SKIP, SKIP, INCLUDE);
323     testDropDeletes(row2, row3, new byte[][] { row1, row1, row2 }, INCLUDE, INCLUDE, SKIP);
324     testDropDeletes(row2, row3, new byte[][] { row2, row3, row3 }, SKIP, INCLUDE, INCLUDE);
325     testDropDeletes(row1, row3, new byte[][] { row1, row2, row3 }, SKIP, SKIP, INCLUDE);
326     // Open ranges.
327     testDropDeletes(HConstants.EMPTY_START_ROW, row3,
328         new byte[][] { row1, row2, row3 }, SKIP, SKIP, INCLUDE);
329     testDropDeletes(row2, HConstants.EMPTY_END_ROW,
330         new byte[][] { row1, row2, row3 }, INCLUDE, SKIP, SKIP);
331     testDropDeletes(HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW,
332         new byte[][] { row1, row2, row3, row3 }, SKIP, SKIP, SKIP, SKIP);
333 
334     // No KVs in range.
335     testDropDeletes(row2, row3, new byte[][] { row1, row1, row3 }, INCLUDE, INCLUDE, INCLUDE);
336     testDropDeletes(row2, row3, new byte[][] { row3, row3 }, INCLUDE, INCLUDE);
337     testDropDeletes(row2, row3, new byte[][] { row1, row1 }, INCLUDE, INCLUDE);
338   }
339 
340   private void testDropDeletes(
341       byte[] from, byte[] to, byte[][] rows, MatchCode... expected) throws IOException {
342     long now = EnvironmentEdgeManager.currentTimeMillis();
343     // Set time to purge deletes to negative value to avoid it ever happening.
344     ScanInfo scanInfo = new ScanInfo(fam2, 0, 1, ttl, KeepDeletedCells.FALSE, -1L, rowComparator);
345     NavigableSet<byte[]> cols = get.getFamilyMap().get(fam2);
346 
347     ScanQueryMatcher qm = new ScanQueryMatcher(scan, scanInfo, cols, Long.MAX_VALUE,
348         HConstants.OLDEST_TIMESTAMP, HConstants.OLDEST_TIMESTAMP, now, from, to, null);
349     List<ScanQueryMatcher.MatchCode> actual =
350         new ArrayList<ScanQueryMatcher.MatchCode>(rows.length);
351     byte[] prevRow = null;
352     for (byte[] row : rows) {
353       if (prevRow == null || !Bytes.equals(prevRow, row)) {
354         qm.setRow(row, 0, (short)row.length);
355         prevRow = row;
356       }
357       actual.add(qm.match(new KeyValue(row, fam2, null, now, Type.Delete)));
358     }
359 
360     assertEquals(expected.length, actual.size());
361     for (int i = 0; i < expected.length; i++) {
362       if (PRINT) System.out.println("expected " + expected[i] + ", actual " + actual.get(i));
363       assertEquals(expected[i], actual.get(i));
364     }
365   }
366 }
367