1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.regionserver;
21
22 import java.io.IOException;
23 import java.util.NavigableSet;
24
25 import org.apache.hadoop.hbase.classification.InterfaceAudience;
26 import org.apache.hadoop.hbase.Cell;
27 import org.apache.hadoop.hbase.CellUtil;
28 import org.apache.hadoop.hbase.HConstants;
29 import org.apache.hadoop.hbase.KeepDeletedCells;
30 import org.apache.hadoop.hbase.KeyValue;
31 import org.apache.hadoop.hbase.client.Scan;
32 import org.apache.hadoop.hbase.filter.Filter;
33 import org.apache.hadoop.hbase.filter.Filter.ReturnCode;
34 import org.apache.hadoop.hbase.io.TimeRange;
35 import org.apache.hadoop.hbase.regionserver.DeleteTracker.DeleteResult;
36 import org.apache.hadoop.hbase.util.Bytes;
37 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
38
39 import com.google.common.base.Preconditions;
40
41
42
43
44 @InterfaceAudience.Private
45 public class ScanQueryMatcher {
46
47
48 private boolean stickyNextRow;
49 private final byte[] stopRow;
50
51 private final TimeRange tr;
52
53 private final Filter filter;
54
55
56 private final DeleteTracker deletes;
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72 private boolean retainDeletesInOutput;
73
74
75 private final KeepDeletedCells keepDeletedCells;
76
77 private final boolean seePastDeleteMarkers;
78
79
80
81 private final ColumnTracker columns;
82
83
84 private final KeyValue startKey;
85
86
87 private final KeyValue.KVComparator rowComparator;
88
89
90
91 byte [] row;
92 int rowOffset;
93 short rowLength;
94
95
96
97
98
99
100
101 private final long earliestPutTs;
102 private final long ttl;
103
104
105 private final long oldestUnexpiredTS;
106 private final long now;
107
108
109 protected long maxReadPointToTrackVersions;
110
111 private byte[] dropDeletesFromRow = null, dropDeletesToRow = null;
112
113
114
115
116
117
118
119 private boolean hasNullColumn = true;
120
121 private RegionCoprocessorHost regionCoprocessorHost= null;
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140 private final long timeToPurgeDeletes;
141
142 private final boolean isUserScan;
143
144 private final boolean isReversed;
145
146
147
148
149
150
151
152
153
154
155
156
157
158 public ScanQueryMatcher(Scan scan, ScanInfo scanInfo, NavigableSet<byte[]> columns,
159 ScanType scanType, long readPointToUse, long earliestPutTs, long oldestUnexpiredTS,
160 long now, RegionCoprocessorHost regionCoprocessorHost) throws IOException {
161 this.tr = scan.getTimeRange();
162 this.rowComparator = scanInfo.getComparator();
163 this.regionCoprocessorHost = regionCoprocessorHost;
164 this.deletes = instantiateDeleteTracker();
165 this.stopRow = scan.getStopRow();
166 this.startKey = KeyValue.createFirstDeleteFamilyOnRow(scan.getStartRow(),
167 scanInfo.getFamily());
168 this.filter = scan.getFilter();
169 this.earliestPutTs = earliestPutTs;
170 this.oldestUnexpiredTS = oldestUnexpiredTS;
171 this.now = now;
172
173 this.maxReadPointToTrackVersions = readPointToUse;
174 this.timeToPurgeDeletes = scanInfo.getTimeToPurgeDeletes();
175 this.ttl = oldestUnexpiredTS;
176
177
178 this.isUserScan = scanType == ScanType.USER_SCAN;
179
180 this.keepDeletedCells = scan.isRaw() ? KeepDeletedCells.TRUE :
181 isUserScan ? KeepDeletedCells.FALSE : scanInfo.getKeepDeletedCells();
182
183 this.retainDeletesInOutput = scanType == ScanType.COMPACT_RETAIN_DELETES || scan.isRaw();
184
185 this.seePastDeleteMarkers =
186 scanInfo.getKeepDeletedCells() != KeepDeletedCells.FALSE && isUserScan;
187
188 int maxVersions =
189 scan.isRaw() ? scan.getMaxVersions() : Math.min(scan.getMaxVersions(),
190 scanInfo.getMaxVersions());
191
192
193 if (columns == null || columns.size() == 0) {
194
195 hasNullColumn = true;
196
197
198 this.columns = new ScanWildcardColumnTracker(
199 scanInfo.getMinVersions(), maxVersions, oldestUnexpiredTS);
200 } else {
201
202 hasNullColumn = (columns.first().length == 0);
203
204
205
206 byte[] attr = scan.getAttribute(Scan.HINT_LOOKAHEAD);
207 this.columns = new ExplicitColumnTracker(columns, scanInfo.getMinVersions(), maxVersions,
208 oldestUnexpiredTS, attr == null ? 0 : Bytes.toInt(attr));
209 }
210 this.isReversed = scan.isReversed();
211 }
212
213 private DeleteTracker instantiateDeleteTracker() throws IOException {
214 DeleteTracker tracker = new ScanDeleteTracker();
215 if (regionCoprocessorHost != null) {
216 tracker = regionCoprocessorHost.postInstantiateDeleteTracker(tracker);
217 }
218 return tracker;
219 }
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234 public ScanQueryMatcher(Scan scan, ScanInfo scanInfo, NavigableSet<byte[]> columns,
235 long readPointToUse, long earliestPutTs, long oldestUnexpiredTS, long now, byte[] dropDeletesFromRow,
236 byte[] dropDeletesToRow, RegionCoprocessorHost regionCoprocessorHost) throws IOException {
237 this(scan, scanInfo, columns, ScanType.COMPACT_RETAIN_DELETES, readPointToUse, earliestPutTs,
238 oldestUnexpiredTS, now, regionCoprocessorHost);
239 Preconditions.checkArgument((dropDeletesFromRow != null) && (dropDeletesToRow != null));
240 this.dropDeletesFromRow = dropDeletesFromRow;
241 this.dropDeletesToRow = dropDeletesToRow;
242 }
243
244
245
246
247 ScanQueryMatcher(Scan scan, ScanInfo scanInfo,
248 NavigableSet<byte[]> columns, long oldestUnexpiredTS, long now) throws IOException {
249 this(scan, scanInfo, columns, ScanType.USER_SCAN,
250 Long.MAX_VALUE,
251 HConstants.LATEST_TIMESTAMP, oldestUnexpiredTS, now, null);
252 }
253
254
255
256
257
258 public boolean hasNullColumnInQuery() {
259 return hasNullColumn;
260 }
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275 public MatchCode match(KeyValue kv) throws IOException {
276 if (filter != null && filter.filterAllRemaining()) {
277 return MatchCode.DONE_SCAN;
278 }
279
280 byte [] bytes = kv.getBuffer();
281 int offset = kv.getOffset();
282
283 int keyLength = Bytes.toInt(bytes, offset, Bytes.SIZEOF_INT);
284 offset += KeyValue.ROW_OFFSET;
285
286 int initialOffset = offset;
287
288 short rowLength = Bytes.toShort(bytes, offset, Bytes.SIZEOF_SHORT);
289 offset += Bytes.SIZEOF_SHORT;
290
291 int ret = this.rowComparator.compareRows(row, this.rowOffset, this.rowLength,
292 bytes, offset, rowLength);
293 if (!this.isReversed) {
294 if (ret <= -1) {
295 return MatchCode.DONE;
296 } else if (ret >= 1) {
297
298
299
300 return MatchCode.SEEK_NEXT_ROW;
301 }
302 } else {
303 if (ret <= -1) {
304 return MatchCode.SEEK_NEXT_ROW;
305 } else if (ret >= 1) {
306 return MatchCode.DONE;
307 }
308 }
309
310
311 if (this.stickyNextRow)
312 return MatchCode.SEEK_NEXT_ROW;
313
314 if (this.columns.done()) {
315 stickyNextRow = true;
316 return MatchCode.SEEK_NEXT_ROW;
317 }
318
319
320 offset += rowLength;
321
322
323 byte familyLength = bytes [offset];
324 offset += familyLength + 1;
325
326 int qualLength = keyLength -
327 (offset - initialOffset) - KeyValue.TIMESTAMP_TYPE_SIZE;
328
329 long timestamp = Bytes.toLong(bytes, initialOffset + keyLength - KeyValue.TIMESTAMP_TYPE_SIZE);
330
331 if (columns.isDone(timestamp)) {
332 return columns.getNextRowOrNextColumn(kv.getQualifierArray(), kv.getQualifierOffset(),
333 kv.getQualifierLength());
334 }
335
336 if (HStore.isCellTTLExpired(kv, this.oldestUnexpiredTS, this.now)) {
337 return MatchCode.SKIP;
338 }
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353 byte type = bytes[initialOffset + keyLength - 1];
354 if (kv.isDelete()) {
355 if (keepDeletedCells == KeepDeletedCells.FALSE
356 || (keepDeletedCells == KeepDeletedCells.TTL && timestamp < ttl)) {
357
358
359
360
361
362
363 boolean includeDeleteMarker = seePastDeleteMarkers ?
364 tr.withinTimeRange(timestamp) :
365 tr.withinOrAfterTimeRange(timestamp);
366 if (includeDeleteMarker
367 && kv.getMvccVersion() <= maxReadPointToTrackVersions) {
368 this.deletes.add(kv);
369 }
370
371 }
372
373 if ((!isUserScan)
374 && timeToPurgeDeletes > 0
375 && (EnvironmentEdgeManager.currentTimeMillis() - timestamp) <= timeToPurgeDeletes) {
376 return MatchCode.INCLUDE;
377 } else if (retainDeletesInOutput || kv.getMvccVersion() > maxReadPointToTrackVersions) {
378
379
380 if (!isUserScan) {
381
382
383 return MatchCode.INCLUDE;
384 }
385 } else if (keepDeletedCells == KeepDeletedCells.TRUE
386 || (keepDeletedCells == KeepDeletedCells.TTL && timestamp >= ttl)) {
387 if (timestamp < earliestPutTs) {
388
389
390 return columns.getNextRowOrNextColumn(bytes, offset, qualLength);
391 }
392
393
394 } else {
395 return MatchCode.SKIP;
396 }
397
398
399 } else if (!this.deletes.isEmpty()) {
400 DeleteResult deleteResult = deletes.isDeleted(kv);
401 switch (deleteResult) {
402 case FAMILY_DELETED:
403 case COLUMN_DELETED:
404 return columns.getNextRowOrNextColumn(bytes, offset, qualLength);
405 case VERSION_DELETED:
406 case FAMILY_VERSION_DELETED:
407 return MatchCode.SKIP;
408 case NOT_DELETED:
409 break;
410 default:
411 throw new RuntimeException("UNEXPECTED");
412 }
413 }
414
415 int timestampComparison = tr.compare(timestamp);
416 if (timestampComparison >= 1) {
417 return MatchCode.SKIP;
418 } else if (timestampComparison <= -1) {
419 return columns.getNextRowOrNextColumn(bytes, offset, qualLength);
420 }
421
422
423 MatchCode colChecker = columns.checkColumn(bytes, offset, qualLength, type);
424 if (colChecker == MatchCode.INCLUDE) {
425 ReturnCode filterResponse = ReturnCode.SKIP;
426
427 if (filter != null) {
428
429 filterResponse = filter.filterKeyValue(kv);
430 switch (filterResponse) {
431 case SKIP:
432 return MatchCode.SKIP;
433 case NEXT_COL:
434 return columns.getNextRowOrNextColumn(bytes, offset, qualLength);
435 case NEXT_ROW:
436 stickyNextRow = true;
437 return MatchCode.SEEK_NEXT_ROW;
438 case SEEK_NEXT_USING_HINT:
439 return MatchCode.SEEK_NEXT_USING_HINT;
440 default:
441
442 break;
443 }
444 }
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464 colChecker =
465 columns.checkVersions(bytes, offset, qualLength, timestamp, type,
466 kv.getMvccVersion() > maxReadPointToTrackVersions);
467
468 stickyNextRow = colChecker == MatchCode.INCLUDE_AND_SEEK_NEXT_ROW ? true : stickyNextRow;
469 return (filterResponse == ReturnCode.INCLUDE_AND_NEXT_COL &&
470 colChecker == MatchCode.INCLUDE) ? MatchCode.INCLUDE_AND_SEEK_NEXT_COL
471 : colChecker;
472 }
473 stickyNextRow = (colChecker == MatchCode.SEEK_NEXT_ROW) ? true
474 : stickyNextRow;
475 return colChecker;
476 }
477
478
479
480
481 private void checkPartialDropDeleteRange(byte [] row, int offset, short length) {
482
483
484
485
486 if ((dropDeletesFromRow != null)
487 && ((dropDeletesFromRow == HConstants.EMPTY_START_ROW)
488 || (Bytes.compareTo(row, offset, length,
489 dropDeletesFromRow, 0, dropDeletesFromRow.length) >= 0))) {
490 retainDeletesInOutput = false;
491 dropDeletesFromRow = null;
492 }
493
494
495
496 if ((dropDeletesFromRow == null)
497 && (dropDeletesToRow != null) && (dropDeletesToRow != HConstants.EMPTY_END_ROW)
498 && (Bytes.compareTo(row, offset, length,
499 dropDeletesToRow, 0, dropDeletesToRow.length) >= 0)) {
500 retainDeletesInOutput = true;
501 dropDeletesToRow = null;
502 }
503 }
504
505 public boolean moreRowsMayExistAfter(KeyValue kv) {
506 if (this.isReversed) {
507 if (rowComparator.compareRows(kv.getBuffer(), kv.getRowOffset(),
508 kv.getRowLength(), stopRow, 0, stopRow.length) <= 0) {
509 return false;
510 } else {
511 return true;
512 }
513 }
514 if (!Bytes.equals(stopRow , HConstants.EMPTY_END_ROW) &&
515 rowComparator.compareRows(kv.getBuffer(),kv.getRowOffset(),
516 kv.getRowLength(), stopRow, 0, stopRow.length) >= 0) {
517
518
519 return false;
520 } else {
521 return true;
522 }
523 }
524
525
526
527
528
529 public void setRow(byte [] row, int offset, short length) {
530 checkPartialDropDeleteRange(row, offset, length);
531 this.row = row;
532 this.rowOffset = offset;
533 this.rowLength = length;
534 reset();
535 }
536
537 public void reset() {
538 this.deletes.reset();
539 this.columns.reset();
540
541 stickyNextRow = false;
542 }
543
544
545
546
547
548 public KeyValue getStartKey() {
549 return this.startKey;
550 }
551
552
553
554
555
556 Filter getFilter() {
557 return this.filter;
558 }
559
560 public Cell getNextKeyHint(Cell kv) throws IOException {
561 if (filter == null) {
562 return null;
563 } else {
564 return filter.getNextCellHint(kv);
565 }
566 }
567
568 public KeyValue getKeyForNextColumn(KeyValue kv) {
569 ColumnCount nextColumn = columns.getColumnHint();
570 if (nextColumn == null) {
571 return KeyValue.createLastOnRow(
572 kv.getBuffer(), kv.getRowOffset(), kv.getRowLength(),
573 kv.getBuffer(), kv.getFamilyOffset(), kv.getFamilyLength(),
574 kv.getBuffer(), kv.getQualifierOffset(), kv.getQualifierLength());
575 } else {
576 return KeyValue.createFirstOnRow(
577 kv.getBuffer(), kv.getRowOffset(), kv.getRowLength(),
578 kv.getBuffer(), kv.getFamilyOffset(), kv.getFamilyLength(),
579 nextColumn.getBuffer(), nextColumn.getOffset(), nextColumn.getLength());
580 }
581 }
582
583 public KeyValue getKeyForNextRow(KeyValue kv) {
584 return KeyValue.createLastOnRow(
585 kv.getBuffer(), kv.getRowOffset(), kv.getRowLength(),
586 null, 0, 0,
587 null, 0, 0);
588 }
589
590
591 static MatchCode checkColumn(ColumnTracker columnTracker, byte[] bytes, int offset,
592 int length, long ttl, byte type, boolean ignoreCount) throws IOException {
593 MatchCode matchCode = columnTracker.checkColumn(bytes, offset, length, type);
594 if (matchCode == MatchCode.INCLUDE) {
595 return columnTracker.checkVersions(bytes, offset, length, ttl, type, ignoreCount);
596 }
597 return matchCode;
598 }
599
600
601
602
603
604
605
606
607 public static enum MatchCode {
608
609
610
611 INCLUDE,
612
613
614
615
616 SKIP,
617
618
619
620
621 NEXT,
622
623
624
625
626 DONE,
627
628
629
630
631
632
633
634
635 SEEK_NEXT_ROW,
636
637
638
639 SEEK_NEXT_COL,
640
641
642
643
644 DONE_SCAN,
645
646
647
648
649 SEEK_NEXT_USING_HINT,
650
651
652
653
654 INCLUDE_AND_SEEK_NEXT_COL,
655
656
657
658
659 INCLUDE_AND_SEEK_NEXT_ROW,
660 }
661 }