1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.io.hfile;
19
20 import java.io.DataInput;
21 import java.io.IOException;
22 import java.nio.ByteBuffer;
23 import java.util.ArrayList;
24 import java.util.List;
25
26 import org.apache.commons.logging.Log;
27 import org.apache.commons.logging.LogFactory;
28 import org.apache.hadoop.hbase.classification.InterfaceAudience;
29 import org.apache.hadoop.conf.Configuration;
30 import org.apache.hadoop.fs.Path;
31 import org.apache.hadoop.hbase.HConstants;
32 import org.apache.hadoop.hbase.KeyValue;
33 import org.apache.hadoop.hbase.KeyValue.KVComparator;
34 import org.apache.hadoop.hbase.fs.HFileSystem;
35 import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper;
36 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoder;
37 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
38 import org.apache.hadoop.hbase.io.encoding.HFileBlockDecodingContext;
39 import org.apache.hadoop.hbase.io.hfile.HFile.FileInfo;
40 import org.apache.hadoop.hbase.util.ByteBufferUtils;
41 import org.apache.hadoop.hbase.util.Bytes;
42 import org.apache.hadoop.hbase.util.IdLock;
43 import org.apache.hadoop.io.WritableUtils;
44 import org.cloudera.htrace.Trace;
45 import org.cloudera.htrace.TraceScope;
46
47 import com.google.common.annotations.VisibleForTesting;
48
49
50
51
52 @InterfaceAudience.Private
53 public class HFileReaderV2 extends AbstractHFileReader {
54
55 private static final Log LOG = LogFactory.getLog(HFileReaderV2.class);
56
57
58 public static final int MINOR_VERSION_WITH_CHECKSUM = 1;
59
60 public static final int MINOR_VERSION_NO_CHECKSUM = 0;
61
62
63 public static final int PBUF_TRAILER_MINOR_VERSION = 2;
64
65
66
67
68
69 public final static int KEY_VALUE_LEN_SIZE = 2 * Bytes.SIZEOF_INT;
70
71 protected boolean includesMemstoreTS = false;
72 protected boolean decodeMemstoreTS = false;
73 protected boolean shouldIncludeMemstoreTS() {
74 return includesMemstoreTS;
75 }
76
77
78 protected HFileBlock.FSReader fsBlockReader;
79
80
81
82
83
84
85
86 private IdLock offsetLock = new IdLock();
87
88
89
90
91
92 private List<HFileBlock> loadOnOpenBlocks = new ArrayList<HFileBlock>();
93
94
95 static final int MIN_MINOR_VERSION = 0;
96
97
98
99
100 static final int MAX_MINOR_VERSION = 3;
101
102
103 static final int MINOR_VERSION_WITH_FAKED_KEY = 3;
104
105 protected HFileContext hfileContext;
106
107
108
109
110
111
112
113
114
115
116
117
118
119 public HFileReaderV2(final Path path, final FixedFileTrailer trailer,
120 final FSDataInputStreamWrapper fsdis, final long size, final CacheConfig cacheConf,
121 final HFileSystem hfs, final Configuration conf) throws IOException {
122 super(path, trailer, size, cacheConf, hfs, conf);
123 this.conf = conf;
124 trailer.expectMajorVersion(getMajorVersion());
125 validateMinorVersion(path, trailer.getMinorVersion());
126 this.hfileContext = createHFileContext(fsdis, fileSize, hfs, path, trailer);
127 HFileBlock.FSReaderV2 fsBlockReaderV2 = new HFileBlock.FSReaderV2(fsdis, fileSize, hfs, path,
128 hfileContext);
129 this.fsBlockReader = fsBlockReaderV2;
130
131
132 comparator = trailer.createComparator();
133 dataBlockIndexReader = new HFileBlockIndex.BlockIndexReader(comparator,
134 trailer.getNumDataIndexLevels(), this);
135 metaBlockIndexReader = new HFileBlockIndex.BlockIndexReader(
136 KeyValue.RAW_COMPARATOR, 1);
137
138
139
140 HFileBlock.BlockIterator blockIter = fsBlockReaderV2.blockRange(
141 trailer.getLoadOnOpenDataOffset(),
142 fileSize - trailer.getTrailerSize());
143
144
145
146 dataBlockIndexReader.readMultiLevelIndexRoot(
147 blockIter.nextBlockWithBlockType(BlockType.ROOT_INDEX),
148 trailer.getDataIndexCount());
149
150
151 metaBlockIndexReader.readRootIndex(
152 blockIter.nextBlockWithBlockType(BlockType.ROOT_INDEX),
153 trailer.getMetaIndexCount());
154
155
156 fileInfo = new FileInfo();
157 fileInfo.read(blockIter.nextBlockWithBlockType(BlockType.FILE_INFO).getByteStream());
158 lastKey = fileInfo.get(FileInfo.LASTKEY);
159 avgKeyLen = Bytes.toInt(fileInfo.get(FileInfo.AVG_KEY_LEN));
160 avgValueLen = Bytes.toInt(fileInfo.get(FileInfo.AVG_VALUE_LEN));
161 byte [] keyValueFormatVersion =
162 fileInfo.get(HFileWriterV2.KEY_VALUE_VERSION);
163 includesMemstoreTS = keyValueFormatVersion != null &&
164 Bytes.toInt(keyValueFormatVersion) ==
165 HFileWriterV2.KEY_VALUE_VER_WITH_MEMSTORE;
166 fsBlockReaderV2.setIncludesMemstoreTS(includesMemstoreTS);
167 if (includesMemstoreTS) {
168 decodeMemstoreTS = Bytes.toLong(fileInfo.get(HFileWriterV2.MAX_MEMSTORE_TS_KEY)) > 0;
169 }
170
171
172 dataBlockEncoder = HFileDataBlockEncoderImpl.createFromFileInfo(fileInfo);
173 fsBlockReaderV2.setDataBlockEncoder(dataBlockEncoder);
174
175
176 HFileBlock b;
177 while ((b = blockIter.nextBlock()) != null) {
178 loadOnOpenBlocks.add(b);
179 }
180
181
182 if (cacheConf.shouldPrefetchOnOpen()) {
183 PrefetchExecutor.request(path, new Runnable() {
184 public void run() {
185 try {
186 long offset = 0;
187 long end = fileSize - getTrailer().getTrailerSize();
188 HFileBlock prevBlock = null;
189 while (offset < end) {
190 if (Thread.interrupted()) {
191 break;
192 }
193 long onDiskSize = -1;
194 if (prevBlock != null) {
195 onDiskSize = prevBlock.getNextBlockOnDiskSizeWithHeader();
196 }
197 HFileBlock block = readBlock(offset, onDiskSize, true, false, false, false, null);
198 prevBlock = block;
199 offset += block.getOnDiskSizeWithHeader();
200 }
201 } catch (IOException e) {
202
203 if (LOG.isTraceEnabled()) {
204 LOG.trace("Exception encountered while prefetching " + path + ":", e);
205 }
206 } catch (Exception e) {
207
208 LOG.warn("Exception encountered while prefetching " + path + ":", e);
209 } finally {
210 PrefetchExecutor.complete(path);
211 }
212 }
213 });
214 }
215 }
216
217 protected HFileContext createHFileContext(FSDataInputStreamWrapper fsdis, long fileSize,
218 HFileSystem hfs, Path path, FixedFileTrailer trailer) throws IOException {
219 return new HFileContextBuilder()
220 .withIncludesMvcc(this.includesMemstoreTS)
221 .withCompression(this.compressAlgo)
222 .withHBaseCheckSum(trailer.getMinorVersion() >= MINOR_VERSION_WITH_CHECKSUM)
223 .build();
224 }
225
226
227
228
229
230
231
232
233
234
235
236
237
238 @Override
239 public HFileScanner getScanner(boolean cacheBlocks, final boolean pread,
240 final boolean isCompaction) {
241 if (dataBlockEncoder.useEncodedScanner()) {
242 return new EncodedScannerV2(this, cacheBlocks, pread, isCompaction,
243 hfileContext);
244 }
245
246 return new ScannerV2(this, cacheBlocks, pread, isCompaction);
247 }
248
249
250
251
252
253
254
255 @Override
256 public ByteBuffer getMetaBlock(String metaBlockName, boolean cacheBlock)
257 throws IOException {
258 if (trailer.getMetaIndexCount() == 0) {
259 return null;
260 }
261 if (metaBlockIndexReader == null) {
262 throw new IOException("Meta index not loaded");
263 }
264
265 byte[] mbname = Bytes.toBytes(metaBlockName);
266 int block = metaBlockIndexReader.rootBlockContainingKey(mbname, 0,
267 mbname.length);
268 if (block == -1)
269 return null;
270 long blockSize = metaBlockIndexReader.getRootBlockDataSize(block);
271
272
273
274
275 synchronized (metaBlockIndexReader.getRootBlockKey(block)) {
276
277 long metaBlockOffset = metaBlockIndexReader.getRootBlockOffset(block);
278 BlockCacheKey cacheKey = new BlockCacheKey(name, metaBlockOffset,
279 DataBlockEncoding.NONE, BlockType.META);
280
281 cacheBlock &= cacheConf.shouldCacheDataOnRead();
282 if (cacheConf.isBlockCacheEnabled()) {
283 HFileBlock cachedBlock =
284 (HFileBlock) cacheConf.getBlockCache().getBlock(cacheKey, cacheBlock, false, true);
285 if (cachedBlock != null) {
286 assert cachedBlock.isUnpacked() : "Packed block leak.";
287
288
289 return cachedBlock.getBufferWithoutHeader();
290 }
291
292 }
293
294 HFileBlock metaBlock = fsBlockReader.readBlockData(metaBlockOffset,
295 blockSize, -1, true).unpack(hfileContext, fsBlockReader);
296
297
298 if (cacheBlock) {
299 cacheConf.getBlockCache().cacheBlock(cacheKey, metaBlock,
300 cacheConf.isInMemory());
301 }
302
303 return metaBlock.getBufferWithoutHeader();
304 }
305 }
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322 @Override
323 public HFileBlock readBlock(long dataBlockOffset, long onDiskBlockSize,
324 final boolean cacheBlock, boolean pread, final boolean isCompaction,
325 final boolean updateCacheMetrics, BlockType expectedBlockType)
326 throws IOException {
327 if (dataBlockIndexReader == null) {
328 throw new IOException("Block index not loaded");
329 }
330 if (dataBlockOffset < 0
331 || dataBlockOffset >= trailer.getLoadOnOpenDataOffset()) {
332 throw new IOException("Requested block is out of range: "
333 + dataBlockOffset + ", lastDataBlockOffset: "
334 + trailer.getLastDataBlockOffset());
335 }
336
337
338
339
340
341
342 BlockCacheKey cacheKey =
343 new BlockCacheKey(name, dataBlockOffset,
344 dataBlockEncoder.getDataBlockEncoding(),
345 expectedBlockType);
346
347 boolean useLock = false;
348 IdLock.Entry lockEntry = null;
349 TraceScope traceScope = Trace.startSpan("HFileReaderV2.readBlock");
350 try {
351 while (true) {
352 if (useLock) {
353 lockEntry = offsetLock.getLockEntry(dataBlockOffset);
354 }
355
356
357 if (cacheConf.isBlockCacheEnabled()) {
358
359
360 HFileBlock cachedBlock = (HFileBlock) cacheConf.getBlockCache().getBlock(cacheKey,
361 cacheBlock, useLock, updateCacheMetrics);
362 if (cachedBlock != null) {
363 if (cacheConf.shouldCacheCompressed(cachedBlock.getBlockType().getCategory())) {
364 cachedBlock = cachedBlock.unpack(hfileContext, fsBlockReader);
365 }
366 if (Trace.isTracing()) {
367 traceScope.getSpan().addTimelineAnnotation("blockCacheHit");
368 }
369 assert cachedBlock.isUnpacked() : "Packed block leak.";
370 if (cachedBlock.getBlockType().isData()) {
371 HFile.dataBlockReadCnt.incrementAndGet();
372
373
374
375 if (cachedBlock.getDataBlockEncoding() != dataBlockEncoder.getDataBlockEncoding()) {
376 throw new IOException("Cached block under key " + cacheKey + " "
377 + "has wrong encoding: " + cachedBlock.getDataBlockEncoding() + " (expected: "
378 + dataBlockEncoder.getDataBlockEncoding() + ")");
379 }
380 }
381 return cachedBlock;
382 }
383
384 }
385 if (!useLock) {
386
387 useLock = true;
388 continue;
389 }
390 if (Trace.isTracing()) {
391 traceScope.getSpan().addTimelineAnnotation("blockCacheMiss");
392 }
393
394 HFileBlock hfileBlock = fsBlockReader.readBlockData(dataBlockOffset, onDiskBlockSize, -1,
395 pread);
396 validateBlockType(hfileBlock, expectedBlockType);
397 HFileBlock unpacked = hfileBlock.unpack(hfileContext, fsBlockReader);
398 BlockType.BlockCategory category = hfileBlock.getBlockType().getCategory();
399
400
401 if (cacheBlock && cacheConf.shouldCacheBlockOnRead(category)) {
402 cacheConf.getBlockCache().cacheBlock(cacheKey,
403 cacheConf.shouldCacheCompressed(category) ? hfileBlock : unpacked,
404 cacheConf.isInMemory());
405 }
406
407 if (updateCacheMetrics && hfileBlock.getBlockType().isData()) {
408 HFile.dataBlockReadCnt.incrementAndGet();
409 }
410
411 return unpacked;
412 }
413 } finally {
414 traceScope.close();
415 if (lockEntry != null) {
416 offsetLock.releaseLockEntry(lockEntry);
417 }
418 }
419 }
420
421 @Override
422 public boolean hasMVCCInfo() {
423 return includesMemstoreTS && decodeMemstoreTS;
424 }
425
426
427
428
429
430
431
432
433
434
435 private void validateBlockType(HFileBlock block,
436 BlockType expectedBlockType) throws IOException {
437 if (expectedBlockType == null) {
438 return;
439 }
440 BlockType actualBlockType = block.getBlockType();
441 if (actualBlockType == BlockType.ENCODED_DATA &&
442 expectedBlockType == BlockType.DATA) {
443
444
445 return;
446 }
447 if (actualBlockType != expectedBlockType) {
448 throw new IOException("Expected block type " + expectedBlockType + ", " +
449 "but got " + actualBlockType + ": " + block);
450 }
451 }
452
453
454
455
456
457
458 @Override
459 public byte[] getLastKey() {
460 return dataBlockIndexReader.isEmpty() ? null : lastKey;
461 }
462
463
464
465
466
467
468 @Override
469 public byte[] midkey() throws IOException {
470 return dataBlockIndexReader.midkey();
471 }
472
473 @Override
474 public void close() throws IOException {
475 close(cacheConf.shouldEvictOnClose());
476 }
477
478 public void close(boolean evictOnClose) throws IOException {
479 PrefetchExecutor.cancel(path);
480 if (evictOnClose && cacheConf.isBlockCacheEnabled()) {
481 int numEvicted = cacheConf.getBlockCache().evictBlocksByHfileName(name);
482 if (LOG.isTraceEnabled()) {
483 LOG.trace("On close, file=" + name + " evicted=" + numEvicted
484 + " block(s)");
485 }
486 }
487 fsBlockReader.closeStreams();
488 }
489
490
491 @Override
492 HFileBlock.FSReader getUncachedBlockReader() {
493 return fsBlockReader;
494 }
495
496
497 protected abstract static class AbstractScannerV2
498 extends AbstractHFileReader.Scanner {
499 protected HFileBlock block;
500
501
502
503
504
505
506
507
508 protected byte[] nextIndexedKey;
509
510 public AbstractScannerV2(HFileReaderV2 r, boolean cacheBlocks,
511 final boolean pread, final boolean isCompaction) {
512 super(r, cacheBlocks, pread, isCompaction);
513 }
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531 protected int seekTo(byte[] key, int offset, int length, boolean rewind)
532 throws IOException {
533 HFileBlockIndex.BlockIndexReader indexReader =
534 reader.getDataBlockIndexReader();
535 BlockWithScanInfo blockWithScanInfo =
536 indexReader.loadDataBlockWithScanInfo(key, offset, length, block,
537 cacheBlocks, pread, isCompaction);
538 if (blockWithScanInfo == null || blockWithScanInfo.getHFileBlock() == null) {
539
540 return -1;
541 }
542 return loadBlockAndSeekToKey(blockWithScanInfo.getHFileBlock(),
543 blockWithScanInfo.getNextIndexedKey(), rewind, key, offset, length, false);
544 }
545
546 protected abstract ByteBuffer getFirstKeyInBlock(HFileBlock curBlock);
547
548 protected abstract int loadBlockAndSeekToKey(HFileBlock seekToBlock, byte[] nextIndexedKey,
549 boolean rewind, byte[] key, int offset, int length, boolean seekBefore)
550 throws IOException;
551
552 @Override
553 public int seekTo(byte[] key, int offset, int length) throws IOException {
554
555
556 return seekTo(key, offset, length, true);
557 }
558
559 @Override
560 public int reseekTo(byte[] key, int offset, int length) throws IOException {
561 int compared;
562 if (isSeeked()) {
563 compared = compareKey(reader.getComparator(), key, offset, length);
564 if (compared < 1) {
565
566
567 return compared;
568 } else {
569 if (this.nextIndexedKey != null &&
570 (this.nextIndexedKey == HConstants.NO_NEXT_INDEXED_KEY ||
571 reader.getComparator().compareFlatKey(key, offset, length,
572 nextIndexedKey, 0, nextIndexedKey.length) < 0)) {
573
574
575
576 return loadBlockAndSeekToKey(this.block, this.nextIndexedKey,
577 false, key, offset, length, false);
578 }
579 }
580 }
581
582
583 return seekTo(key, offset, length, false);
584 }
585
586 @Override
587 public boolean seekBefore(byte[] key, int offset, int length)
588 throws IOException {
589 HFileBlock seekToBlock =
590 reader.getDataBlockIndexReader().seekToDataBlock(key, offset, length,
591 block, cacheBlocks, pread, isCompaction);
592 if (seekToBlock == null) {
593 return false;
594 }
595 ByteBuffer firstKey = getFirstKeyInBlock(seekToBlock);
596
597 if (reader.getComparator().compareFlatKey(firstKey.array(),
598 firstKey.arrayOffset(), firstKey.limit(), key, offset, length) >= 0)
599 {
600 long previousBlockOffset = seekToBlock.getPrevBlockOffset();
601
602 if (previousBlockOffset == -1) {
603
604 return false;
605 }
606
607
608
609
610 seekToBlock = reader.readBlock(previousBlockOffset,
611 seekToBlock.getOffset() - previousBlockOffset, cacheBlocks,
612 pread, isCompaction, true, BlockType.DATA);
613
614
615 }
616 byte[] firstKeyInCurrentBlock = Bytes.getBytes(firstKey);
617 loadBlockAndSeekToKey(seekToBlock, firstKeyInCurrentBlock, true, key, offset, length, true);
618 return true;
619 }
620
621
622
623
624
625
626
627
628
629 protected HFileBlock readNextDataBlock() throws IOException {
630 long lastDataBlockOffset = reader.getTrailer().getLastDataBlockOffset();
631 if (block == null)
632 return null;
633
634 HFileBlock curBlock = block;
635
636 do {
637 if (curBlock.getOffset() >= lastDataBlockOffset)
638 return null;
639
640 if (curBlock.getOffset() < 0) {
641 throw new IOException("Invalid block file offset: " + block);
642 }
643
644
645
646 curBlock = reader.readBlock(curBlock.getOffset()
647 + curBlock.getOnDiskSizeWithHeader(),
648 curBlock.getNextBlockOnDiskSizeWithHeader(), cacheBlocks, pread,
649 isCompaction, true, null);
650 } while (!curBlock.getBlockType().isData());
651
652 return curBlock;
653 }
654
655
656
657
658
659
660
661
662 public abstract int compareKey(KVComparator comparator, byte[] key, int offset,
663 int length);
664 }
665
666
667
668
669 protected static class ScannerV2 extends AbstractScannerV2 {
670 private HFileReaderV2 reader;
671
672 public ScannerV2(HFileReaderV2 r, boolean cacheBlocks,
673 final boolean pread, final boolean isCompaction) {
674 super(r, cacheBlocks, pread, isCompaction);
675 this.reader = r;
676 }
677
678 @Override
679 public KeyValue getKeyValue() {
680 if (!isSeeked())
681 return null;
682
683 KeyValue ret = new KeyValue(blockBuffer.array(), blockBuffer.arrayOffset()
684 + blockBuffer.position(), getCellBufSize());
685 if (this.reader.shouldIncludeMemstoreTS()) {
686 ret.setMvccVersion(currMemstoreTS);
687 }
688 return ret;
689 }
690
691 protected int getCellBufSize() {
692 return KEY_VALUE_LEN_SIZE + currKeyLen + currValueLen;
693 }
694
695 @Override
696 public ByteBuffer getKey() {
697 assertSeeked();
698 return ByteBuffer.wrap(
699 blockBuffer.array(),
700 blockBuffer.arrayOffset() + blockBuffer.position()
701 + KEY_VALUE_LEN_SIZE, currKeyLen).slice();
702 }
703
704 @Override
705 public int compareKey(KVComparator comparator, byte[] key, int offset, int length) {
706 return comparator.compareFlatKey(key, offset, length, blockBuffer.array(),
707 blockBuffer.arrayOffset() + blockBuffer.position() + KEY_VALUE_LEN_SIZE, currKeyLen);
708 }
709
710 @Override
711 public ByteBuffer getValue() {
712 assertSeeked();
713 return ByteBuffer.wrap(
714 blockBuffer.array(),
715 blockBuffer.arrayOffset() + blockBuffer.position()
716 + KEY_VALUE_LEN_SIZE + currKeyLen, currValueLen).slice();
717 }
718
719 protected void setNonSeekedState() {
720 block = null;
721 blockBuffer = null;
722 currKeyLen = 0;
723 currValueLen = 0;
724 currMemstoreTS = 0;
725 currMemstoreTSLen = 0;
726 }
727
728
729
730
731
732
733
734
735 @Override
736 public boolean next() throws IOException {
737 assertSeeked();
738
739 try {
740 blockBuffer.position(getNextCellStartPosition());
741 } catch (IllegalArgumentException e) {
742 LOG.error("Current pos = " + blockBuffer.position()
743 + "; currKeyLen = " + currKeyLen + "; currValLen = "
744 + currValueLen + "; block limit = " + blockBuffer.limit()
745 + "; HFile name = " + reader.getName()
746 + "; currBlock currBlockOffset = " + block.getOffset());
747 throw e;
748 }
749
750 if (blockBuffer.remaining() <= 0) {
751 long lastDataBlockOffset =
752 reader.getTrailer().getLastDataBlockOffset();
753
754 if (block.getOffset() >= lastDataBlockOffset) {
755 setNonSeekedState();
756 return false;
757 }
758
759
760 HFileBlock nextBlock = readNextDataBlock();
761 if (nextBlock == null) {
762 setNonSeekedState();
763 return false;
764 }
765
766 updateCurrBlock(nextBlock);
767 return true;
768 }
769
770
771 readKeyValueLen();
772 return true;
773 }
774
775 protected int getNextCellStartPosition() {
776 return blockBuffer.position() + KEY_VALUE_LEN_SIZE + currKeyLen + currValueLen
777 + currMemstoreTSLen;
778 }
779
780
781
782
783
784
785
786
787 @Override
788 public boolean seekTo() throws IOException {
789 if (reader == null) {
790 return false;
791 }
792
793 if (reader.getTrailer().getEntryCount() == 0) {
794
795 return false;
796 }
797
798 long firstDataBlockOffset =
799 reader.getTrailer().getFirstDataBlockOffset();
800 if (block != null && block.getOffset() == firstDataBlockOffset) {
801 blockBuffer.rewind();
802 readKeyValueLen();
803 return true;
804 }
805
806 block = reader.readBlock(firstDataBlockOffset, -1, cacheBlocks, pread,
807 isCompaction, true, BlockType.DATA);
808 if (block.getOffset() < 0) {
809 throw new IOException("Invalid block offset: " + block.getOffset());
810 }
811 updateCurrBlock(block);
812 return true;
813 }
814
815 @Override
816 protected int loadBlockAndSeekToKey(HFileBlock seekToBlock, byte[] nextIndexedKey,
817 boolean rewind, byte[] key, int offset, int length, boolean seekBefore)
818 throws IOException {
819 if (block == null || block.getOffset() != seekToBlock.getOffset()) {
820 updateCurrBlock(seekToBlock);
821 } else if (rewind) {
822 blockBuffer.rewind();
823 }
824
825
826 this.nextIndexedKey = nextIndexedKey;
827 return blockSeek(key, offset, length, seekBefore);
828 }
829
830
831
832
833
834
835
836 protected void updateCurrBlock(HFileBlock newBlock) {
837 block = newBlock;
838
839
840 if (block.getBlockType() != BlockType.DATA) {
841 throw new IllegalStateException("ScannerV2 works only on data " +
842 "blocks, got " + block.getBlockType() + "; " +
843 "fileName=" + reader.name + ", " +
844 "dataBlockEncoder=" + reader.dataBlockEncoder + ", " +
845 "isCompaction=" + isCompaction);
846 }
847
848 blockBuffer = block.getBufferWithoutHeader();
849 readKeyValueLen();
850 blockFetches++;
851
852
853 this.nextIndexedKey = null;
854 }
855
856 protected void readKeyValueLen() {
857 blockBuffer.mark();
858 currKeyLen = blockBuffer.getInt();
859 currValueLen = blockBuffer.getInt();
860 ByteBufferUtils.skip(blockBuffer, currKeyLen + currValueLen);
861 readMvccVersion();
862 if (currKeyLen < 0 || currValueLen < 0
863 || currKeyLen > blockBuffer.limit()
864 || currValueLen > blockBuffer.limit()) {
865 throw new IllegalStateException("Invalid currKeyLen " + currKeyLen
866 + " or currValueLen " + currValueLen + ". Block offset: "
867 + block.getOffset() + ", block length: " + blockBuffer.limit()
868 + ", position: " + blockBuffer.position() + " (without header).");
869 }
870 blockBuffer.reset();
871 }
872
873 protected void readMvccVersion() {
874 if (this.reader.shouldIncludeMemstoreTS()) {
875 if (this.reader.decodeMemstoreTS) {
876 try {
877 currMemstoreTS = Bytes.readVLong(blockBuffer.array(), blockBuffer.arrayOffset()
878 + blockBuffer.position());
879 currMemstoreTSLen = WritableUtils.getVIntSize(currMemstoreTS);
880 } catch (Exception e) {
881 throw new RuntimeException("Error reading memstore timestamp", e);
882 }
883 } else {
884 currMemstoreTS = 0;
885 currMemstoreTSLen = 1;
886 }
887 }
888 }
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905 protected int blockSeek(byte[] key, int offset, int length,
906 boolean seekBefore) {
907 int klen, vlen;
908 long memstoreTS = 0;
909 int memstoreTSLen = 0;
910 int lastKeyValueSize = -1;
911 do {
912 blockBuffer.mark();
913 klen = blockBuffer.getInt();
914 vlen = blockBuffer.getInt();
915 blockBuffer.reset();
916 if (this.reader.shouldIncludeMemstoreTS()) {
917 if (this.reader.decodeMemstoreTS) {
918 try {
919 int memstoreTSOffset = blockBuffer.arrayOffset()
920 + blockBuffer.position() + KEY_VALUE_LEN_SIZE + klen + vlen;
921 memstoreTS = Bytes.readVLong(blockBuffer.array(),
922 memstoreTSOffset);
923 memstoreTSLen = WritableUtils.getVIntSize(memstoreTS);
924 } catch (Exception e) {
925 throw new RuntimeException("Error reading memstore timestamp", e);
926 }
927 } else {
928 memstoreTS = 0;
929 memstoreTSLen = 1;
930 }
931 }
932
933 int keyOffset = blockBuffer.arrayOffset() + blockBuffer.position()
934 + KEY_VALUE_LEN_SIZE;
935 int comp = reader.getComparator().compareFlatKey(key, offset, length,
936 blockBuffer.array(), keyOffset, klen);
937
938 if (comp == 0) {
939 if (seekBefore) {
940 if (lastKeyValueSize < 0) {
941 throw new IllegalStateException("blockSeek with seekBefore "
942 + "at the first key of the block: key="
943 + Bytes.toStringBinary(key) + ", blockOffset="
944 + block.getOffset() + ", onDiskSize="
945 + block.getOnDiskSizeWithHeader());
946 }
947 blockBuffer.position(blockBuffer.position() - lastKeyValueSize);
948 readKeyValueLen();
949 return 1;
950 }
951 currKeyLen = klen;
952 currValueLen = vlen;
953 if (this.reader.shouldIncludeMemstoreTS()) {
954 currMemstoreTS = memstoreTS;
955 currMemstoreTSLen = memstoreTSLen;
956 }
957 return 0;
958 } else if (comp < 0) {
959 if (lastKeyValueSize > 0)
960 blockBuffer.position(blockBuffer.position() - lastKeyValueSize);
961 readKeyValueLen();
962 if (lastKeyValueSize == -1 && blockBuffer.position() == 0
963 && this.reader.trailer.getMinorVersion() >= MINOR_VERSION_WITH_FAKED_KEY) {
964 return HConstants.INDEX_KEY_MAGIC;
965 }
966 return 1;
967 }
968
969
970 lastKeyValueSize = klen + vlen + memstoreTSLen + KEY_VALUE_LEN_SIZE;
971 blockBuffer.position(blockBuffer.position() + lastKeyValueSize);
972 } while (blockBuffer.remaining() > 0);
973
974
975
976
977 blockBuffer.position(blockBuffer.position() - lastKeyValueSize);
978 readKeyValueLen();
979 return 1;
980 }
981
982 @Override
983 protected ByteBuffer getFirstKeyInBlock(HFileBlock curBlock) {
984 ByteBuffer buffer = curBlock.getBufferWithoutHeader();
985
986 buffer.rewind();
987 int klen = buffer.getInt();
988 buffer.getInt();
989 ByteBuffer keyBuff = buffer.slice();
990 keyBuff.limit(klen);
991 keyBuff.rewind();
992 return keyBuff;
993 }
994
995 @Override
996 public String getKeyString() {
997 return Bytes.toStringBinary(blockBuffer.array(),
998 blockBuffer.arrayOffset() + blockBuffer.position()
999 + KEY_VALUE_LEN_SIZE, currKeyLen);
1000 }
1001
1002 @Override
1003 public String getValueString() {
1004 return Bytes.toString(blockBuffer.array(), blockBuffer.arrayOffset()
1005 + blockBuffer.position() + KEY_VALUE_LEN_SIZE + currKeyLen,
1006 currValueLen);
1007 }
1008 }
1009
1010
1011
1012
1013 protected static class EncodedScannerV2 extends AbstractScannerV2 {
1014 private final HFileBlockDecodingContext decodingCtx;
1015 private final DataBlockEncoder.EncodedSeeker seeker;
1016 private final DataBlockEncoder dataBlockEncoder;
1017 protected final HFileContext meta;
1018
1019 public EncodedScannerV2(HFileReaderV2 reader, boolean cacheBlocks,
1020 boolean pread, boolean isCompaction, HFileContext meta) {
1021 super(reader, cacheBlocks, pread, isCompaction);
1022 DataBlockEncoding encoding = reader.dataBlockEncoder.getDataBlockEncoding();
1023 dataBlockEncoder = encoding.getEncoder();
1024 decodingCtx = dataBlockEncoder.newDataBlockDecodingContext(meta);
1025 seeker = dataBlockEncoder.createSeeker(
1026 reader.getComparator(), decodingCtx);
1027 this.meta = meta;
1028 }
1029
1030 @Override
1031 public boolean isSeeked(){
1032 return this.block != null;
1033 }
1034
1035
1036
1037
1038
1039
1040
1041
1042 private void updateCurrentBlock(HFileBlock newBlock) throws CorruptHFileException {
1043 block = newBlock;
1044
1045
1046 if (block.getBlockType() != BlockType.ENCODED_DATA) {
1047 throw new IllegalStateException(
1048 "EncodedScanner works only on encoded data blocks");
1049 }
1050 short dataBlockEncoderId = block.getDataBlockEncodingId();
1051 if (!DataBlockEncoding.isCorrectEncoder(dataBlockEncoder, dataBlockEncoderId)) {
1052 String encoderCls = dataBlockEncoder.getClass().getName();
1053 throw new CorruptHFileException("Encoder " + encoderCls
1054 + " doesn't support data block encoding "
1055 + DataBlockEncoding.getNameFromId(dataBlockEncoderId));
1056 }
1057
1058 seeker.setCurrentBuffer(getEncodedBuffer(newBlock));
1059 blockFetches++;
1060
1061
1062 this.nextIndexedKey = null;
1063 }
1064
1065 private ByteBuffer getEncodedBuffer(HFileBlock newBlock) {
1066 ByteBuffer origBlock = newBlock.getBufferReadOnly();
1067 ByteBuffer encodedBlock = ByteBuffer.wrap(origBlock.array(),
1068 origBlock.arrayOffset() + newBlock.headerSize() +
1069 DataBlockEncoding.ID_SIZE,
1070 newBlock.getUncompressedSizeWithoutHeader() -
1071 DataBlockEncoding.ID_SIZE).slice();
1072 return encodedBlock;
1073 }
1074
1075 @Override
1076 public boolean seekTo() throws IOException {
1077 if (reader == null) {
1078 return false;
1079 }
1080
1081 if (reader.getTrailer().getEntryCount() == 0) {
1082
1083 return false;
1084 }
1085
1086 long firstDataBlockOffset =
1087 reader.getTrailer().getFirstDataBlockOffset();
1088 if (block != null && block.getOffset() == firstDataBlockOffset) {
1089 seeker.rewind();
1090 return true;
1091 }
1092
1093 block = reader.readBlock(firstDataBlockOffset, -1, cacheBlocks, pread,
1094 isCompaction, true, BlockType.DATA);
1095 if (block.getOffset() < 0) {
1096 throw new IOException("Invalid block offset: " + block.getOffset());
1097 }
1098 updateCurrentBlock(block);
1099 return true;
1100 }
1101
1102 @Override
1103 public boolean next() throws IOException {
1104 boolean isValid = seeker.next();
1105 if (!isValid) {
1106 block = readNextDataBlock();
1107 isValid = block != null;
1108 if (isValid) {
1109 updateCurrentBlock(block);
1110 }
1111 }
1112 return isValid;
1113 }
1114
1115 @Override
1116 public ByteBuffer getKey() {
1117 assertValidSeek();
1118 return seeker.getKeyDeepCopy();
1119 }
1120
1121 @Override
1122 public int compareKey(KVComparator comparator, byte[] key, int offset, int length) {
1123 return seeker.compareKey(comparator, key, offset, length);
1124 }
1125
1126 @Override
1127 public ByteBuffer getValue() {
1128 assertValidSeek();
1129 return seeker.getValueShallowCopy();
1130 }
1131
1132 @Override
1133 public KeyValue getKeyValue() {
1134 if (block == null) {
1135 return null;
1136 }
1137 return seeker.getKeyValue();
1138 }
1139
1140 @Override
1141 public String getKeyString() {
1142 ByteBuffer keyBuffer = getKey();
1143 return Bytes.toStringBinary(keyBuffer.array(),
1144 keyBuffer.arrayOffset(), keyBuffer.limit());
1145 }
1146
1147 @Override
1148 public String getValueString() {
1149 ByteBuffer valueBuffer = getValue();
1150 return Bytes.toStringBinary(valueBuffer.array(),
1151 valueBuffer.arrayOffset(), valueBuffer.limit());
1152 }
1153
1154 private void assertValidSeek() {
1155 if (block == null) {
1156 throw new NotSeekedException();
1157 }
1158 }
1159
1160 @Override
1161 protected ByteBuffer getFirstKeyInBlock(HFileBlock curBlock) {
1162 return dataBlockEncoder.getFirstKeyInBlock(getEncodedBuffer(curBlock));
1163 }
1164
1165 @Override
1166 protected int loadBlockAndSeekToKey(HFileBlock seekToBlock, byte[] nextIndexedKey,
1167 boolean rewind, byte[] key, int offset, int length, boolean seekBefore)
1168 throws IOException {
1169 if (block == null || block.getOffset() != seekToBlock.getOffset()) {
1170 updateCurrentBlock(seekToBlock);
1171 } else if (rewind) {
1172 seeker.rewind();
1173 }
1174 this.nextIndexedKey = nextIndexedKey;
1175 return seeker.seekToKeyInBlock(key, offset, length, seekBefore);
1176 }
1177 }
1178
1179
1180
1181
1182
1183 @Override
1184 public DataInput getGeneralBloomFilterMetadata() throws IOException {
1185 return this.getBloomFilterMetadata(BlockType.GENERAL_BLOOM_META);
1186 }
1187
1188 @Override
1189 public DataInput getDeleteBloomFilterMetadata() throws IOException {
1190 return this.getBloomFilterMetadata(BlockType.DELETE_FAMILY_BLOOM_META);
1191 }
1192
1193 private DataInput getBloomFilterMetadata(BlockType blockType)
1194 throws IOException {
1195 if (blockType != BlockType.GENERAL_BLOOM_META &&
1196 blockType != BlockType.DELETE_FAMILY_BLOOM_META) {
1197 throw new RuntimeException("Block Type: " + blockType.toString() +
1198 " is not supported") ;
1199 }
1200
1201 for (HFileBlock b : loadOnOpenBlocks)
1202 if (b.getBlockType() == blockType)
1203 return b.getByteStream();
1204 return null;
1205 }
1206
1207 @Override
1208 public boolean isFileInfoLoaded() {
1209 return true;
1210 }
1211
1212
1213
1214
1215
1216 private void validateMinorVersion(Path path, int minorVersion) {
1217 if (minorVersion < MIN_MINOR_VERSION ||
1218 minorVersion > MAX_MINOR_VERSION) {
1219 String msg = "Minor version for path " + path +
1220 " is expected to be between " +
1221 MIN_MINOR_VERSION + " and " + MAX_MINOR_VERSION +
1222 " but is found to be " + minorVersion;
1223 LOG.error(msg);
1224 throw new RuntimeException(msg);
1225 }
1226 }
1227
1228 @Override
1229 public int getMajorVersion() {
1230 return 2;
1231 }
1232
1233 @Override
1234 public HFileContext getFileContext() {
1235 return hfileContext;
1236 }
1237
1238
1239
1240
1241
1242 @VisibleForTesting
1243 boolean prefetchComplete() {
1244 return PrefetchExecutor.isCompleted(path);
1245 }
1246 }