1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.filter;
20
21
22 import java.util.ArrayList;
23 import java.util.Arrays;
24 import java.util.List;
25
26 import static org.junit.Assert.assertEquals;
27 import static org.junit.Assert.assertFalse;
28 import static org.junit.Assert.assertTrue;
29 import static org.junit.Assert.assertNull;
30
31 import org.apache.hadoop.hbase.Cell;
32 import org.apache.hadoop.hbase.KeyValue;
33 import org.apache.hadoop.hbase.KeyValueUtil;
34 import org.apache.hadoop.hbase.exceptions.DeserializationException;
35 import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
36 import org.apache.hadoop.hbase.filter.Filter.ReturnCode;
37 import org.apache.hadoop.hbase.filter.FilterList.Operator;
38 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
39 import org.apache.hadoop.hbase.testclassification.SmallTests;
40 import org.apache.hadoop.hbase.util.Bytes;
41 import org.junit.Test;
42 import org.junit.experimental.categories.Category;
43
44 import com.google.common.collect.Lists;
45
46
47
48
49
50 @Category(SmallTests.class)
51 public class TestFilterList {
52 static final int MAX_PAGES = 2;
53 static final char FIRST_CHAR = 'a';
54 static final char LAST_CHAR = 'e';
55 static byte[] GOOD_BYTES = Bytes.toBytes("abc");
56 static byte[] BAD_BYTES = Bytes.toBytes("def");
57
58
59 @Test
60 public void testAddFilter() throws Exception {
61 Filter filter1 = new FirstKeyOnlyFilter();
62 Filter filter2 = new FirstKeyOnlyFilter();
63
64 FilterList filterList = new FilterList(filter1, filter2);
65 filterList.addFilter(new FirstKeyOnlyFilter());
66
67 filterList = new FilterList(Arrays.asList(filter1, filter2));
68 filterList.addFilter(new FirstKeyOnlyFilter());
69
70 filterList = new FilterList(Operator.MUST_PASS_ALL, filter1, filter2);
71 filterList.addFilter(new FirstKeyOnlyFilter());
72
73 filterList = new FilterList(Operator.MUST_PASS_ALL, Arrays.asList(filter1, filter2));
74 filterList.addFilter(new FirstKeyOnlyFilter());
75
76 }
77
78
79
80
81
82
83 @Test
84 public void testMPONE() throws Exception {
85 mpOneTest(getFilterMPONE());
86 }
87
88 private Filter getFilterMPONE() {
89 List<Filter> filters = new ArrayList<Filter>();
90 filters.add(new PageFilter(MAX_PAGES));
91 filters.add(new WhileMatchFilter(new PrefixFilter(Bytes.toBytes("yyy"))));
92 Filter filterMPONE =
93 new FilterList(FilterList.Operator.MUST_PASS_ONE, filters);
94 return filterMPONE;
95 }
96
97 private void mpOneTest(Filter filterMPONE) throws Exception {
98
99
100
101
102
103
104
105
106
107
108
109
110 filterMPONE.reset();
111 assertFalse(filterMPONE.filterAllRemaining());
112
113
114 byte [] rowkey = Bytes.toBytes("yyyyyyyyy");
115 for (int i = 0; i < MAX_PAGES - 1; i++) {
116 assertFalse(filterMPONE.filterRowKey(rowkey, 0, rowkey.length));
117 KeyValue kv = new KeyValue(rowkey, rowkey, Bytes.toBytes(i),
118 Bytes.toBytes(i));
119 assertTrue(Filter.ReturnCode.INCLUDE == filterMPONE.filterKeyValue(kv));
120 assertFalse(filterMPONE.filterRow());
121 }
122
123
124 rowkey = Bytes.toBytes("z");
125 assertFalse(filterMPONE.filterRowKey(rowkey, 0, rowkey.length));
126 KeyValue kv = new KeyValue(rowkey, rowkey, Bytes.toBytes(0),
127 Bytes.toBytes(0));
128 assertTrue(Filter.ReturnCode.INCLUDE == filterMPONE.filterKeyValue(kv));
129 assertFalse(filterMPONE.filterRow());
130
131
132 rowkey = Bytes.toBytes("yyy");
133 assertTrue(filterMPONE.filterRowKey(rowkey, 0, rowkey.length));
134 kv = new KeyValue(rowkey, rowkey, Bytes.toBytes(0),
135 Bytes.toBytes(0));
136 assertFalse(Filter.ReturnCode.INCLUDE == filterMPONE.filterKeyValue(kv));
137 assertFalse(filterMPONE.filterRow());
138
139
140 rowkey = Bytes.toBytes("z");
141 assertTrue(filterMPONE.filterRowKey(rowkey, 0, rowkey.length));
142 assertTrue(filterMPONE.filterAllRemaining());
143 }
144
145
146
147
148
149 @Test
150 public void testMPALL() throws Exception {
151 mpAllTest(getMPALLFilter());
152 }
153
154 private Filter getMPALLFilter() {
155 List<Filter> filters = new ArrayList<Filter>();
156 filters.add(new PageFilter(MAX_PAGES));
157 filters.add(new WhileMatchFilter(new PrefixFilter(Bytes.toBytes("yyy"))));
158 Filter filterMPALL =
159 new FilterList(FilterList.Operator.MUST_PASS_ALL, filters);
160 return filterMPALL;
161 }
162
163 private void mpAllTest(Filter filterMPALL) throws Exception {
164
165
166
167
168
169
170
171
172
173
174
175
176 filterMPALL.reset();
177 assertFalse(filterMPALL.filterAllRemaining());
178 byte [] rowkey = Bytes.toBytes("yyyyyyyyy");
179 for (int i = 0; i < MAX_PAGES - 1; i++) {
180 assertFalse(filterMPALL.filterRowKey(rowkey, 0, rowkey.length));
181 KeyValue kv = new KeyValue(rowkey, rowkey, Bytes.toBytes(i),
182 Bytes.toBytes(i));
183 assertTrue(Filter.ReturnCode.INCLUDE == filterMPALL.filterKeyValue(kv));
184 }
185 filterMPALL.reset();
186 rowkey = Bytes.toBytes("z");
187 assertTrue(filterMPALL.filterRowKey(rowkey, 0, rowkey.length));
188
189 KeyValue kv = new KeyValue(rowkey, rowkey, rowkey, rowkey);
190 assertTrue(Filter.ReturnCode.NEXT_ROW == filterMPALL.filterKeyValue(kv));
191 }
192
193
194
195
196
197 @Test
198 public void testOrdering() throws Exception {
199 orderingTest(getOrderingFilter());
200 }
201
202 public Filter getOrderingFilter() {
203 List<Filter> filters = new ArrayList<Filter>();
204 filters.add(new PrefixFilter(Bytes.toBytes("yyy")));
205 filters.add(new PageFilter(MAX_PAGES));
206 Filter filterMPONE =
207 new FilterList(FilterList.Operator.MUST_PASS_ONE, filters);
208 return filterMPONE;
209 }
210
211 public void orderingTest(Filter filterMPONE) throws Exception {
212
213
214
215
216
217
218
219
220
221
222
223
224 filterMPONE.reset();
225 assertFalse(filterMPONE.filterAllRemaining());
226
227
228 byte [] rowkey = Bytes.toBytes("yyyyyyyy");
229 for (int i = 0; i < MAX_PAGES; i++) {
230 assertFalse(filterMPONE.filterRowKey(rowkey, 0, rowkey.length));
231 KeyValue kv = new KeyValue(rowkey, rowkey, Bytes.toBytes(i),
232 Bytes.toBytes(i));
233 assertTrue(Filter.ReturnCode.INCLUDE == filterMPONE.filterKeyValue(kv));
234 assertFalse(filterMPONE.filterRow());
235 }
236
237
238 rowkey = Bytes.toBytes("xxxxxxx");
239 for (int i = 0; i < MAX_PAGES; i++) {
240 assertFalse(filterMPONE.filterRowKey(rowkey, 0, rowkey.length));
241 KeyValue kv = new KeyValue(rowkey, rowkey, Bytes.toBytes(i),
242 Bytes.toBytes(i));
243 assertTrue(Filter.ReturnCode.INCLUDE == filterMPONE.filterKeyValue(kv));
244 assertFalse(filterMPONE.filterRow());
245 }
246
247
248 rowkey = Bytes.toBytes("yyy");
249 for (int i = 0; i < MAX_PAGES; i++) {
250 assertFalse(filterMPONE.filterRowKey(rowkey, 0, rowkey.length));
251 KeyValue kv = new KeyValue(rowkey, rowkey, Bytes.toBytes(i),
252 Bytes.toBytes(i));
253 assertTrue(Filter.ReturnCode.INCLUDE == filterMPONE.filterKeyValue(kv));
254 assertFalse(filterMPONE.filterRow());
255 }
256 }
257
258
259
260
261
262
263 public void testFilterListTwoFiltersMustPassOne() throws Exception {
264 byte[] r1 = Bytes.toBytes("Row1");
265 byte[] r11 = Bytes.toBytes("Row11");
266 byte[] r2 = Bytes.toBytes("Row2");
267
268 FilterList flist = new FilterList(FilterList.Operator.MUST_PASS_ONE);
269 flist.addFilter(new PrefixFilter(r1));
270 flist.filterRowKey(r1, 0, r1.length);
271 assertEquals(flist.filterKeyValue(new KeyValue(r1,r1,r1)), ReturnCode.INCLUDE);
272 assertEquals(flist.filterKeyValue(new KeyValue(r11,r11,r11)), ReturnCode.INCLUDE);
273
274 flist.reset();
275 flist.filterRowKey(r2, 0, r2.length);
276 assertEquals(flist.filterKeyValue(new KeyValue(r2,r2,r2)), ReturnCode.SKIP);
277
278 flist = new FilterList(FilterList.Operator.MUST_PASS_ONE);
279 flist.addFilter(new AlwaysNextColFilter());
280 flist.addFilter(new PrefixFilter(r1));
281 flist.filterRowKey(r1, 0, r1.length);
282 assertEquals(flist.filterKeyValue(new KeyValue(r1,r1,r1)), ReturnCode.INCLUDE);
283 assertEquals(flist.filterKeyValue(new KeyValue(r11,r11,r11)), ReturnCode.INCLUDE);
284
285 flist.reset();
286 flist.filterRowKey(r2, 0, r2.length);
287 assertEquals(flist.filterKeyValue(new KeyValue(r2,r2,r2)), ReturnCode.SKIP);
288 }
289
290
291
292
293
294
295 public void testFilterListWithInclusiveStopFilteMustPassOne() throws Exception {
296 byte[] r1 = Bytes.toBytes("Row1");
297 byte[] r11 = Bytes.toBytes("Row11");
298 byte[] r2 = Bytes.toBytes("Row2");
299
300 FilterList flist = new FilterList(FilterList.Operator.MUST_PASS_ONE);
301 flist.addFilter(new AlwaysNextColFilter());
302 flist.addFilter(new InclusiveStopFilter(r1));
303 flist.filterRowKey(r1, 0, r1.length);
304 assertEquals(flist.filterKeyValue(new KeyValue(r1,r1,r1)), ReturnCode.INCLUDE);
305 assertEquals(flist.filterKeyValue(new KeyValue(r11,r11,r11)), ReturnCode.INCLUDE);
306
307 flist.reset();
308 flist.filterRowKey(r2, 0, r2.length);
309 assertEquals(flist.filterKeyValue(new KeyValue(r2,r2,r2)), ReturnCode.SKIP);
310 }
311
312 public static class AlwaysNextColFilter extends FilterBase {
313 public AlwaysNextColFilter() {
314 super();
315 }
316 @Override
317 public ReturnCode filterKeyValue(Cell v) {
318 return ReturnCode.NEXT_COL;
319 }
320 public static AlwaysNextColFilter parseFrom(final byte [] pbBytes)
321 throws DeserializationException {
322 return new AlwaysNextColFilter();
323 }
324 }
325
326
327
328
329
330 @Test
331 public void testSerialization() throws Exception {
332 List<Filter> filters = new ArrayList<Filter>();
333 filters.add(new PageFilter(MAX_PAGES));
334 filters.add(new WhileMatchFilter(new PrefixFilter(Bytes.toBytes("yyy"))));
335 Filter filterMPALL =
336 new FilterList(FilterList.Operator.MUST_PASS_ALL, filters);
337
338
339 byte[] buffer = filterMPALL.toByteArray();
340
341
342 FilterList newFilter = FilterList.parseFrom(buffer);
343
344
345 mpOneTest(ProtobufUtil.toFilter(ProtobufUtil.toFilter(getFilterMPONE())));
346 mpAllTest(ProtobufUtil.toFilter(ProtobufUtil.toFilter(getMPALLFilter())));
347 orderingTest(ProtobufUtil.toFilter(ProtobufUtil.toFilter(getOrderingFilter())));
348 }
349
350
351
352
353
354 public void testFilterKeyValue() throws Exception {
355 Filter includeFilter = new FilterBase() {
356 @Override
357 public Filter.ReturnCode filterKeyValue(Cell v) {
358 return Filter.ReturnCode.INCLUDE;
359 }
360 };
361
362 Filter alternateFilter = new FilterBase() {
363 boolean returnInclude = true;
364
365 @Override
366 public Filter.ReturnCode filterKeyValue(Cell v) {
367 Filter.ReturnCode returnCode = returnInclude ? Filter.ReturnCode.INCLUDE :
368 Filter.ReturnCode.SKIP;
369 returnInclude = !returnInclude;
370 return returnCode;
371 }
372 };
373
374 Filter alternateIncludeFilter = new FilterBase() {
375 boolean returnIncludeOnly = false;
376
377 @Override
378 public Filter.ReturnCode filterKeyValue(Cell v) {
379 Filter.ReturnCode returnCode = returnIncludeOnly ? Filter.ReturnCode.INCLUDE :
380 Filter.ReturnCode.INCLUDE_AND_NEXT_COL;
381 returnIncludeOnly = !returnIncludeOnly;
382 return returnCode;
383 }
384 };
385
386
387 FilterList mpOnefilterList = new FilterList(Operator.MUST_PASS_ONE,
388 Arrays.asList(new Filter[] { includeFilter, alternateIncludeFilter, alternateFilter }));
389
390 assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL, mpOnefilterList.filterKeyValue(null));
391
392 assertEquals(Filter.ReturnCode.INCLUDE, mpOnefilterList.filterKeyValue(null));
393
394
395 FilterList mpAllfilterList = new FilterList(Operator.MUST_PASS_ALL,
396 Arrays.asList(new Filter[] { includeFilter, alternateIncludeFilter, alternateFilter }));
397
398 assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL, mpAllfilterList.filterKeyValue(null));
399
400 assertEquals(Filter.ReturnCode.SKIP, mpAllfilterList.filterKeyValue(null));
401 }
402
403
404
405
406 @Test
407 public void testHintPassThru() throws Exception {
408
409 final KeyValue minKeyValue = new KeyValue(Bytes.toBytes(0L), null, null);
410 final KeyValue maxKeyValue = new KeyValue(Bytes.toBytes(Long.MAX_VALUE),
411 null, null);
412
413 Filter filterNoHint = new FilterBase() {
414 @Override
415 public byte [] toByteArray() {return null;}
416 };
417
418 Filter filterMinHint = new FilterBase() {
419 @Override
420 public ReturnCode filterKeyValue(Cell ignored) {
421 return ReturnCode.SEEK_NEXT_USING_HINT;
422 }
423
424 @Override
425 public Cell getNextCellHint(Cell currentKV) {
426 return minKeyValue;
427 }
428
429 @Override
430 public byte [] toByteArray() {return null;}
431 };
432
433 Filter filterMaxHint = new FilterBase() {
434 @Override
435 public ReturnCode filterKeyValue(Cell ignored) {
436 return ReturnCode.SEEK_NEXT_USING_HINT;
437 }
438
439 @Override
440 public Cell getNextCellHint(Cell currentKV) {
441 return new KeyValue(Bytes.toBytes(Long.MAX_VALUE), null, null);
442 }
443
444 @Override
445 public byte [] toByteArray() {return null;}
446 };
447
448
449
450
451 FilterList filterList = new FilterList(Operator.MUST_PASS_ONE,
452 Arrays.asList(new Filter [] { filterMinHint, filterMaxHint } ));
453 assertEquals(0, KeyValue.COMPARATOR.compare(filterList.getNextKeyHint(null),
454 minKeyValue));
455
456
457 filterList = new FilterList(Operator.MUST_PASS_ONE,
458 Arrays.asList(
459 new Filter [] { filterMinHint, filterMaxHint, filterNoHint } ));
460 assertNull(filterList.getNextKeyHint(null));
461 filterList = new FilterList(Operator.MUST_PASS_ONE,
462 Arrays.asList(new Filter [] { filterNoHint, filterMaxHint } ));
463 assertNull(filterList.getNextKeyHint(null));
464
465
466 filterList = new FilterList(Operator.MUST_PASS_ONE,
467 Arrays.asList(new Filter [] { filterMaxHint, filterMaxHint } ));
468 assertEquals(0, KeyValue.COMPARATOR.compare(filterList.getNextKeyHint(null),
469 maxKeyValue));
470
471
472
473
474 filterList = new FilterList(Operator.MUST_PASS_ALL,
475 Arrays.asList(new Filter [] { filterMinHint, filterMaxHint } ));
476 filterList.filterKeyValue(null);
477 assertEquals(0, KeyValue.COMPARATOR.compare(filterList.getNextKeyHint(null),
478 minKeyValue));
479
480 filterList = new FilterList(Operator.MUST_PASS_ALL,
481 Arrays.asList(new Filter [] { filterMaxHint, filterMinHint } ));
482 filterList.filterKeyValue(null);
483 assertEquals(0, KeyValue.COMPARATOR.compare(filterList.getNextKeyHint(null),
484 maxKeyValue));
485
486
487 filterList = new FilterList(Operator.MUST_PASS_ALL,
488 Arrays.asList(
489 new Filter [] { filterNoHint, filterMinHint, filterMaxHint } ));
490 filterList.filterKeyValue(null);
491 assertEquals(0, KeyValue.COMPARATOR.compare(filterList.getNextKeyHint(null),
492 minKeyValue));
493 filterList = new FilterList(Operator.MUST_PASS_ALL,
494 Arrays.asList(new Filter [] { filterNoHint, filterMaxHint } ));
495 filterList.filterKeyValue(null);
496 assertEquals(0, KeyValue.COMPARATOR.compare(filterList.getNextKeyHint(null),
497 maxKeyValue));
498 filterList = new FilterList(Operator.MUST_PASS_ALL,
499 Arrays.asList(new Filter [] { filterNoHint, filterMinHint } ));
500 filterList.filterKeyValue(null);
501 assertEquals(0, KeyValue.COMPARATOR.compare(filterList.getNextKeyHint(null),
502 minKeyValue));
503 }
504
505
506
507
508
509
510
511 @Test
512 public void testTransformMPO() throws Exception {
513
514
515
516 final FilterList flist = new FilterList(Operator.MUST_PASS_ONE, Lists.<Filter>newArrayList(
517 new FilterList(Operator.MUST_PASS_ALL, Lists.<Filter>newArrayList(
518 new FamilyFilter(CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes("fam"))),
519 new QualifierFilter(CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes("qual1"))),
520 new KeyOnlyFilter())),
521 new FilterList(Operator.MUST_PASS_ALL, Lists.<Filter>newArrayList(
522 new FamilyFilter(CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes("fam"))),
523 new QualifierFilter(CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes("qual2")))))));
524
525 final KeyValue kvQual1 = new KeyValue(
526 Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("qual1"), Bytes.toBytes("value"));
527 final KeyValue kvQual2 = new KeyValue(
528 Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("qual2"), Bytes.toBytes("value"));
529 final KeyValue kvQual3 = new KeyValue(
530 Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("qual3"), Bytes.toBytes("value"));
531
532
533 assertEquals(Filter.ReturnCode.INCLUDE, flist.filterKeyValue(kvQual1));
534 final KeyValue transformedQual1 = KeyValueUtil.ensureKeyValue(flist.transform(kvQual1));
535 assertEquals(0, transformedQual1.getValue().length);
536
537
538 assertEquals(Filter.ReturnCode.INCLUDE, flist.filterKeyValue(kvQual2));
539 final KeyValue transformedQual2 = KeyValueUtil.ensureKeyValue(flist.transform(kvQual2));
540 assertEquals("value", Bytes.toString(transformedQual2.getValue()));
541
542
543 assertEquals(Filter.ReturnCode.SKIP, flist.filterKeyValue(kvQual3));
544 }
545
546 }
547