View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.codec.prefixtree.decode;
20  
21  import org.apache.hadoop.hbase.classification.InterfaceAudience;
22  import org.apache.hadoop.hbase.Cell;
23  import org.apache.hadoop.hbase.CellComparator;
24  import org.apache.hadoop.hbase.CellScanner;
25  import org.apache.hadoop.hbase.codec.prefixtree.PrefixTreeBlockMeta;
26  import org.apache.hadoop.hbase.codec.prefixtree.decode.column.ColumnReader;
27  import org.apache.hadoop.hbase.codec.prefixtree.decode.row.RowNodeReader;
28  import org.apache.hadoop.hbase.codec.prefixtree.decode.timestamp.MvccVersionDecoder;
29  import org.apache.hadoop.hbase.codec.prefixtree.decode.timestamp.TimestampDecoder;
30  import org.apache.hadoop.hbase.codec.prefixtree.encode.other.ColumnNodeType;
31  import org.apache.hadoop.hbase.util.Bytes;
32  
33  /**
34   * Extends PtCell and manipulates its protected fields.  Could alternatively contain a PtCell and
35   * call get/set methods.
36   *
37   * This is an "Array" scanner to distinguish from a future "ByteBuffer" scanner.  This
38   * implementation requires that the bytes be in a normal java byte[] for performance.  The
39   * alternative ByteBuffer implementation would allow for accessing data in an off-heap ByteBuffer
40   * without copying the whole buffer on-heap.
41   */
42  @InterfaceAudience.Private
43  public class PrefixTreeArrayScanner extends PrefixTreeCell implements CellScanner {
44  
45    /***************** fields ********************************/
46  
47    protected PrefixTreeBlockMeta blockMeta;
48  
49    protected boolean beforeFirst;
50    protected boolean afterLast;
51  
52    protected RowNodeReader[] rowNodes;
53    protected int rowNodeStackIndex;
54  
55    protected RowNodeReader currentRowNode;
56    protected ColumnReader familyReader;
57    protected ColumnReader qualifierReader;
58    protected ColumnReader tagsReader;
59    protected TimestampDecoder timestampDecoder;
60    protected MvccVersionDecoder mvccVersionDecoder;
61  
62    protected boolean nubCellsRemain;
63    protected int currentCellIndex;
64  
65  
66    /*********************** construct ******************************/
67  
68    // pass in blockMeta so we can initialize buffers big enough for all cells in the block
69    public PrefixTreeArrayScanner(PrefixTreeBlockMeta blockMeta, int rowTreeDepth,
70        int rowBufferLength, int qualifierBufferLength, int tagsBufferLength) {
71      this.rowNodes = new RowNodeReader[rowTreeDepth];
72      for (int i = 0; i < rowNodes.length; ++i) {
73        rowNodes[i] = new RowNodeReader();
74      }
75      this.rowBuffer = new byte[rowBufferLength];
76      this.familyBuffer = new byte[PrefixTreeBlockMeta.MAX_FAMILY_LENGTH];
77      this.familyReader = new ColumnReader(familyBuffer, ColumnNodeType.FAMILY);
78      this.qualifierBuffer = new byte[qualifierBufferLength];
79      this.tagsBuffer = new byte[tagsBufferLength];
80      this.qualifierReader = new ColumnReader(qualifierBuffer, ColumnNodeType.QUALIFIER);
81      this.tagsReader = new ColumnReader(tagsBuffer, ColumnNodeType.TAGS);
82      this.timestampDecoder = new TimestampDecoder();
83      this.mvccVersionDecoder = new MvccVersionDecoder();
84    }
85  
86  
87    /**************** init helpers ***************************************/
88  
89    /**
90     * Call when first accessing a block.
91     * @return entirely new scanner if false
92     */
93    public boolean areBuffersBigEnough() {
94      if (rowNodes.length < blockMeta.getRowTreeDepth()) {
95        return false;
96      }
97      if (rowBuffer.length < blockMeta.getMaxRowLength()) {
98        return false;
99      }
100     if (qualifierBuffer.length < blockMeta.getMaxQualifierLength()) {
101       return false;
102     }
103     if(tagsBuffer.length < blockMeta.getMaxTagsLength()) {
104       return false;
105     }
106     return true;
107   }
108 
109   public void initOnBlock(PrefixTreeBlockMeta blockMeta, byte[] block,
110       boolean includeMvccVersion) {
111     this.block = block;
112     this.blockMeta = blockMeta;
113     this.familyOffset = familyBuffer.length;
114     this.familyReader.initOnBlock(blockMeta, block);
115     this.qualifierOffset = qualifierBuffer.length;
116     this.qualifierReader.initOnBlock(blockMeta, block);
117     this.tagsOffset = tagsBuffer.length;
118     this.tagsReader.initOnBlock(blockMeta, block);
119     this.timestampDecoder.initOnBlock(blockMeta, block);
120     this.mvccVersionDecoder.initOnBlock(blockMeta, block);
121     this.includeMvccVersion = includeMvccVersion;
122     resetToBeforeFirstEntry();
123   }
124 
125   // Does this have to be in the CellScanner Interface?  TODO
126   public void resetToBeforeFirstEntry() {
127     beforeFirst = true;
128     afterLast = false;
129     rowNodeStackIndex = -1;
130     currentRowNode = null;
131     rowLength = 0;
132     familyOffset = familyBuffer.length;
133     familyLength = 0;
134     qualifierOffset = blockMeta.getMaxQualifierLength();
135     qualifierLength = 0;
136     nubCellsRemain = false;
137     currentCellIndex = -1;
138     timestamp = -1L;
139     type = DEFAULT_TYPE;
140     absoluteValueOffset = 0;//use 0 vs -1 so the cell is valid when value hasn't been initialized
141     valueLength = 0;// had it at -1, but that causes null Cell to add up to the wrong length
142     tagsOffset = blockMeta.getMaxTagsLength();
143     tagsLength = 0;
144   }
145 
146   /**
147    * Call this before putting the scanner back into a pool so it doesn't hold the last used block
148    * in memory.
149    */
150   public void releaseBlockReference(){
151     block = null;
152   }
153 
154 
155   /********************** CellScanner **********************/
156 
157   @Override
158   public Cell current() {
159     if(isOutOfBounds()){
160       return null;
161     }
162     return (Cell)this;
163   }
164 
165   /******************* Object methods ************************/
166 
167   @Override
168   public boolean equals(Object obj) {
169     //trivial override to confirm intent (findbugs)
170     return super.equals(obj);
171   }
172   
173   @Override
174   public int hashCode() {
175     return super.hashCode();
176   }
177 
178   /**
179    * Override PrefixTreeCell.toString() with a check to see if the current cell is valid.
180    */
181   @Override
182   public String toString() {
183     Cell currentCell = current();
184     if(currentCell==null){
185       return "null";
186     }
187     return ((PrefixTreeCell)currentCell).getKeyValueString();
188   }
189 
190 
191   /******************* advance ***************************/
192 
193   public boolean positionAtFirstCell() {
194     reInitFirstNode();
195     return advance();
196   }
197 
198   @Override
199   public boolean advance() {
200     if (afterLast) {
201       return false;
202     }
203     if (!hasOccurrences()) {
204       resetToBeforeFirstEntry();
205     }
206     if (beforeFirst || isLastCellInRow()) {
207       nextRow();
208       if (afterLast) {
209         return false;
210       }
211     } else {
212       ++currentCellIndex;
213     }
214 
215     populateNonRowFields(currentCellIndex);
216     return true;
217   }
218 
219 
220   public boolean nextRow() {
221     nextRowInternal();
222     if (afterLast) {
223       return false;
224     }
225     populateNonRowFields(currentCellIndex);
226     return true;
227   }
228 
229 
230   /**
231    * This method is safe to call when the scanner is not on a fully valid row node, as in the case
232    * of a row token miss in the Searcher
233    * @return true if we are positioned on a valid row, false if past end of block
234    */
235   protected boolean nextRowInternal() {
236     if (afterLast) {
237       return false;
238     }
239     if (beforeFirst) {
240       initFirstNode();
241       if (currentRowNode.hasOccurrences()) {
242         if (currentRowNode.isNub()) {
243           nubCellsRemain = true;
244         }
245         currentCellIndex = 0;
246         return true;
247       }
248     }
249     if (currentRowNode.isLeaf()) {
250       discardCurrentRowNode(true);
251     }
252     while (!afterLast) {
253       if (nubCellsRemain) {
254         nubCellsRemain = false;
255       }
256       if (currentRowNode.hasMoreFanNodes()) {
257         followNextFan();
258         if (currentRowNode.hasOccurrences()) {
259           // found some values
260           currentCellIndex = 0;
261           return true;
262         }
263       } else {
264         discardCurrentRowNode(true);
265       }
266     }
267     return false;// went past the end
268   }
269 
270 
271   /**************** secondary traversal methods ******************************/
272 
273   protected void reInitFirstNode() {
274     resetToBeforeFirstEntry();
275     initFirstNode();
276   }
277 
278   protected void initFirstNode() {
279     int offsetIntoUnderlyingStructure = blockMeta.getAbsoluteRowOffset();
280     rowNodeStackIndex = 0;
281     currentRowNode = rowNodes[0];
282     currentRowNode.initOnBlock(blockMeta, block, offsetIntoUnderlyingStructure);
283     appendCurrentTokenToRowBuffer();
284     beforeFirst = false;
285   }
286 
287   protected void followFirstFan() {
288     followFan(0);
289   }
290 
291   protected void followPreviousFan() {
292     int nextFanPosition = currentRowNode.getFanIndex() - 1;
293     followFan(nextFanPosition);
294   }
295 
296   protected void followCurrentFan() {
297     int currentFanPosition = currentRowNode.getFanIndex();
298     followFan(currentFanPosition);
299   }
300 
301   protected void followNextFan() {
302     int nextFanPosition = currentRowNode.getFanIndex() + 1;
303     followFan(nextFanPosition);
304   }
305 
306   protected void followLastFan() {
307     followFan(currentRowNode.getLastFanIndex());
308   }
309 
310   protected void followFan(int fanIndex) {
311     currentRowNode.setFanIndex(fanIndex);
312     appendToRowBuffer(currentRowNode.getFanByte(fanIndex));
313 
314     int nextOffsetIntoUnderlyingStructure = currentRowNode.getOffset()
315         + currentRowNode.getNextNodeOffset(fanIndex, blockMeta);
316     ++rowNodeStackIndex;
317 
318     currentRowNode = rowNodes[rowNodeStackIndex];
319     currentRowNode.initOnBlock(blockMeta, block, nextOffsetIntoUnderlyingStructure);
320 
321     //TODO getToken is spewing garbage
322     appendCurrentTokenToRowBuffer();
323     if (currentRowNode.isNub()) {
324       nubCellsRemain = true;
325     }
326     currentCellIndex = 0;
327   }
328 
329   /**
330    * @param forwards which marker to set if we overflow
331    */
332   protected void discardCurrentRowNode(boolean forwards) {
333     RowNodeReader rowNodeBeingPopped = currentRowNode;
334     --rowNodeStackIndex;// pop it off the stack
335     if (rowNodeStackIndex < 0) {
336       currentRowNode = null;
337       if (forwards) {
338         markAfterLast();
339       } else {
340         markBeforeFirst();
341       }
342       return;
343     }
344     popFromRowBuffer(rowNodeBeingPopped);
345     currentRowNode = rowNodes[rowNodeStackIndex];
346   }
347 
348   protected void markBeforeFirst() {
349     beforeFirst = true;
350     afterLast = false;
351     currentRowNode = null;
352   }
353 
354   protected void markAfterLast() {
355     beforeFirst = false;
356     afterLast = true;
357     currentRowNode = null;
358   }
359 
360 
361   /***************** helper methods **************************/
362 
363   protected void appendCurrentTokenToRowBuffer() {
364     System.arraycopy(block, currentRowNode.getTokenArrayOffset(), rowBuffer, rowLength, 
365       currentRowNode.getTokenLength());
366     rowLength += currentRowNode.getTokenLength();
367   }
368 
369   protected void appendToRowBuffer(byte b) {
370     rowBuffer[rowLength] = b;
371     ++rowLength;
372   }
373 
374   protected void popFromRowBuffer(RowNodeReader rowNodeBeingPopped) {
375     rowLength -= rowNodeBeingPopped.getTokenLength();
376     --rowLength; // pop the parent's fan byte
377   }
378 
379   protected boolean hasOccurrences() {
380     return currentRowNode != null && currentRowNode.hasOccurrences();
381   }
382 
383   protected boolean isBranch() {
384     return currentRowNode != null && !currentRowNode.hasOccurrences()
385         && currentRowNode.hasChildren();
386   }
387 
388   protected boolean isNub() {
389     return currentRowNode != null && currentRowNode.hasOccurrences()
390         && currentRowNode.hasChildren();
391   }
392 
393   protected boolean isLeaf() {
394     return currentRowNode != null && currentRowNode.hasOccurrences()
395         && !currentRowNode.hasChildren();
396   }
397 
398   //TODO expose this in a PrefixTreeScanner interface
399   public boolean isBeforeFirst(){
400     return beforeFirst;
401   }
402 
403   public boolean isAfterLast(){
404     return afterLast;
405   }
406 
407   protected boolean isOutOfBounds(){
408     return beforeFirst || afterLast;
409   }
410 
411   protected boolean isFirstCellInRow() {
412     return currentCellIndex == 0;
413   }
414 
415   protected boolean isLastCellInRow() {
416     return currentCellIndex == currentRowNode.getLastCellIndex();
417   }
418 
419 
420   /********************* fill in family/qualifier/ts/type/value ************/
421 
422   protected int populateNonRowFieldsAndCompareTo(int cellNum, Cell key) {
423     populateNonRowFields(cellNum);
424     return CellComparator.compareStatic(this, key, true);
425   }
426 
427   protected void populateFirstNonRowFields() {
428     populateNonRowFields(0);
429   }
430 
431   protected void populatePreviousNonRowFields() {
432     populateNonRowFields(currentCellIndex - 1);
433   }
434 
435   protected void populateLastNonRowFields() {
436     populateNonRowFields(currentRowNode.getLastCellIndex());
437   }
438 
439   protected void populateNonRowFields(int cellIndex) {
440     currentCellIndex = cellIndex;
441     populateFamily();
442     populateQualifier();
443     // Read tags only if there are tags in the meta
444     if(blockMeta.getNumTagsBytes() != 0) {
445       populateTag();
446     }
447     populateTimestamp();
448     populateMvccVersion();
449     populateType();
450     populateValueOffsets();
451   }
452 
453   protected void populateFamily() {
454     int familyTreeIndex = currentRowNode.getFamilyOffset(currentCellIndex, blockMeta);
455     familyOffset = familyReader.populateBuffer(familyTreeIndex).getColumnOffset();
456     familyLength = familyReader.getColumnLength();
457   }
458 
459   protected void populateQualifier() {
460     int qualifierTreeIndex = currentRowNode.getColumnOffset(currentCellIndex, blockMeta);
461     qualifierOffset = qualifierReader.populateBuffer(qualifierTreeIndex).getColumnOffset();
462     qualifierLength = qualifierReader.getColumnLength();
463   }
464 
465   protected void populateTag() {
466     int tagTreeIndex = currentRowNode.getTagOffset(currentCellIndex, blockMeta);
467     tagsOffset = tagsReader.populateBuffer(tagTreeIndex).getColumnOffset();
468     tagsLength = tagsReader.getColumnLength();
469   }
470 
471   protected void populateTimestamp() {
472     if (blockMeta.isAllSameTimestamp()) {
473       timestamp = blockMeta.getMinTimestamp();
474     } else {
475       int timestampIndex = currentRowNode.getTimestampIndex(currentCellIndex, blockMeta);
476       timestamp = timestampDecoder.getLong(timestampIndex);
477     }
478   }
479 
480   protected void populateMvccVersion() {
481     if (blockMeta.isAllSameMvccVersion()) {
482       mvccVersion = blockMeta.getMinMvccVersion();
483     } else {
484       int mvccVersionIndex = currentRowNode.getMvccVersionIndex(currentCellIndex,
485         blockMeta);
486       mvccVersion = mvccVersionDecoder.getMvccVersion(mvccVersionIndex);
487     }
488   }
489 
490   protected void populateType() {
491     int typeInt;
492     if (blockMeta.isAllSameType()) {
493       typeInt = blockMeta.getAllTypes();
494     } else {
495       typeInt = currentRowNode.getType(currentCellIndex, blockMeta);
496     }
497     type = PrefixTreeCell.TYPES[typeInt];
498   }
499 
500   protected void populateValueOffsets() {
501     int offsetIntoValueSection = currentRowNode.getValueOffset(currentCellIndex, blockMeta);
502     absoluteValueOffset = blockMeta.getAbsoluteValueOffset() + offsetIntoValueSection;
503     valueLength = currentRowNode.getValueLength(currentCellIndex, blockMeta);
504   }
505 
506   /**************** getters ***************************/
507 
508   public byte[] getTreeBytes() {
509     return block;
510   }
511 
512   public PrefixTreeBlockMeta getBlockMeta() {
513     return blockMeta;
514   }
515 
516   public int getMaxRowTreeStackNodes() {
517     return rowNodes.length;
518   }
519 
520   public int getRowBufferLength() {
521     return rowBuffer.length;
522   }
523 
524   public int getQualifierBufferLength() {
525     return qualifierBuffer.length;
526   }
527 
528   public int getTagBufferLength() {
529     return tagsBuffer.length;
530   }
531 }