View Javadoc

1   /**
2    * Copyright The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase;
21  
22  import static org.apache.hadoop.hbase.util.Bytes.len;
23  
24  import java.io.DataInput;
25  import java.io.DataOutput;
26  import java.io.IOException;
27  import java.io.InputStream;
28  import java.io.OutputStream;
29  import java.nio.ByteBuffer;
30  import java.util.ArrayList;
31  import java.util.Arrays;
32  import java.util.Comparator;
33  import java.util.HashMap;
34  import java.util.List;
35  import java.util.Map;
36  
37  import org.apache.commons.logging.Log;
38  import org.apache.commons.logging.LogFactory;
39  import org.apache.hadoop.hbase.classification.InterfaceAudience;
40  import org.apache.hadoop.hbase.io.HeapSize;
41  import org.apache.hadoop.hbase.util.Bytes;
42  import org.apache.hadoop.hbase.util.ClassSize;
43  import org.apache.hadoop.io.IOUtils;
44  import org.apache.hadoop.io.RawComparator;
45  
46  import com.google.common.primitives.Longs;
47  
48  /**
49   * An HBase Key/Value. This is the fundamental HBase Type.  
50   * <p>
51   * HBase applications and users should use the Cell interface and avoid directly using KeyValue
52   * and member functions not defined in Cell.
53   * <p>
54   * If being used client-side, the primary methods to access individual fields are {@link #getRow()},
55   * {@link #getFamily()}, {@link #getQualifier()}, {@link #getTimestamp()}, and {@link #getValue()}.
56   * These methods allocate new byte arrays and return copies. Avoid their use server-side.
57   * <p>
58   * Instances of this class are immutable. They do not implement Comparable but Comparators are
59   * provided. Comparators change with context, whether user table or a catalog table comparison. Its
60   * critical you use the appropriate comparator. There are Comparators for normal HFiles, Meta's
61   * Hfiles, and bloom filter keys.
62   * <p>
63   * KeyValue wraps a byte array and takes offsets and lengths into passed array at where to start
64   * interpreting the content as KeyValue. The KeyValue format inside a byte array is:
65   * <code>&lt;keylength> &lt;valuelength> &lt;key> &lt;value></code> Key is further decomposed as:
66   * <code>&lt;rowlength> &lt;row> &lt;columnfamilylength> &lt;columnfamily> &lt;columnqualifier>
67   * &lt;timestamp> &lt;keytype></code>
68   * The <code>rowlength</code> maximum is <code>Short.MAX_SIZE</code>, column family length maximum
69   * is <code>Byte.MAX_SIZE</code>, and column qualifier + key length must be <
70   * <code>Integer.MAX_SIZE</code>. The column does not contain the family/qualifier delimiter,
71   * {@link #COLUMN_FAMILY_DELIMITER}<br>
72   * KeyValue can optionally contain Tags. When it contains tags, it is added in the byte array after
73   * the value part. The format for this part is: <code>&lt;tagslength>&lt;tagsbytes></code>.
74   * <code>tagslength</code> maximum is <code>Short.MAX_SIZE</code>. The <code>tagsbytes</code>
75   * contain one or more tags where as each tag is of the form
76   * <code>&lt;taglength>&lt;tagtype>&lt;tagbytes></code>.  <code>tagtype</code> is one byte and
77   * <code>taglength</code> maximum is <code>Short.MAX_SIZE</code> and it includes 1 byte type length
78   * and actual tag bytes length.
79   */
80  @InterfaceAudience.Private
81  public class KeyValue implements Cell, HeapSize, Cloneable {
82    private static final ArrayList<Tag> EMPTY_ARRAY_LIST = new ArrayList<Tag>();
83  
84    static final Log LOG = LogFactory.getLog(KeyValue.class);
85  
86    /**
87     * Colon character in UTF-8
88     */
89    public static final char COLUMN_FAMILY_DELIMITER = ':';
90  
91    public static final byte[] COLUMN_FAMILY_DELIM_ARRAY =
92      new byte[]{COLUMN_FAMILY_DELIMITER};
93  
94    /**
95     * Comparator for plain key/values; i.e. non-catalog table key/values. Works on Key portion
96     * of KeyValue only.
97     */
98    public static final KVComparator COMPARATOR = new KVComparator();
99    /**
100    * A {@link KVComparator} for <code>hbase:meta</code> catalog table
101    * {@link KeyValue}s.
102    */
103   public static final KVComparator META_COMPARATOR = new MetaComparator();
104 
105   /**
106    * Needed for Bloom Filters.
107    */
108   public static final KVComparator RAW_COMPARATOR = new RawBytesComparator();
109 
110   /** Size of the key length field in bytes*/
111   public static final int KEY_LENGTH_SIZE = Bytes.SIZEOF_INT;
112 
113   /** Size of the key type field in bytes */
114   public static final int TYPE_SIZE = Bytes.SIZEOF_BYTE;
115 
116   /** Size of the row length field in bytes */
117   public static final int ROW_LENGTH_SIZE = Bytes.SIZEOF_SHORT;
118 
119   /** Size of the family length field in bytes */
120   public static final int FAMILY_LENGTH_SIZE = Bytes.SIZEOF_BYTE;
121 
122   /** Size of the timestamp field in bytes */
123   public static final int TIMESTAMP_SIZE = Bytes.SIZEOF_LONG;
124 
125   // Size of the timestamp and type byte on end of a key -- a long + a byte.
126   public static final int TIMESTAMP_TYPE_SIZE = TIMESTAMP_SIZE + TYPE_SIZE;
127 
128   // Size of the length shorts and bytes in key.
129   public static final int KEY_INFRASTRUCTURE_SIZE = ROW_LENGTH_SIZE
130       + FAMILY_LENGTH_SIZE + TIMESTAMP_TYPE_SIZE;
131 
132   // How far into the key the row starts at. First thing to read is the short
133   // that says how long the row is.
134   public static final int ROW_OFFSET =
135     Bytes.SIZEOF_INT /*keylength*/ +
136     Bytes.SIZEOF_INT /*valuelength*/;
137 
138   // Size of the length ints in a KeyValue datastructure.
139   public static final int KEYVALUE_INFRASTRUCTURE_SIZE = ROW_OFFSET;
140 
141   /** Size of the tags length field in bytes */
142   public static final int TAGS_LENGTH_SIZE = Bytes.SIZEOF_SHORT;
143 
144   public static final int KEYVALUE_WITH_TAGS_INFRASTRUCTURE_SIZE = ROW_OFFSET + TAGS_LENGTH_SIZE;
145 
146   private static final int MAX_TAGS_LENGTH = (2 * Short.MAX_VALUE) + 1;
147 
148   /**
149    * Computes the number of bytes that a <code>KeyValue</code> instance with the provided
150    * characteristics would take up for its underlying data structure.
151    *
152    * @param rlength row length
153    * @param flength family length
154    * @param qlength qualifier length
155    * @param vlength value length
156    *
157    * @return the <code>KeyValue</code> data structure length
158    */
159   public static long getKeyValueDataStructureSize(int rlength,
160       int flength, int qlength, int vlength) {
161     return KeyValue.KEYVALUE_INFRASTRUCTURE_SIZE
162         + getKeyDataStructureSize(rlength, flength, qlength) + vlength;
163   }
164 
165   /**
166    * Computes the number of bytes that a <code>KeyValue</code> instance with the provided
167    * characteristics would take up for its underlying data structure.
168    *
169    * @param rlength row length
170    * @param flength family length
171    * @param qlength qualifier length
172    * @param vlength value length
173    * @param tagsLength total length of the tags
174    *
175    * @return the <code>KeyValue</code> data structure length
176    */
177   public static long getKeyValueDataStructureSize(int rlength, int flength, int qlength,
178       int vlength, int tagsLength) {
179     if (tagsLength == 0) {
180       return getKeyValueDataStructureSize(rlength, flength, qlength, vlength);
181     }
182     return KeyValue.KEYVALUE_WITH_TAGS_INFRASTRUCTURE_SIZE
183         + getKeyDataStructureSize(rlength, flength, qlength) + vlength + tagsLength;
184   }
185 
186   /**
187    * Computes the number of bytes that a <code>KeyValue</code> instance with the provided
188    * characteristics would take up for its underlying data structure.
189    *
190    * @param klength key length
191    * @param vlength value length
192    * @param tagsLength total length of the tags
193    *
194    * @return the <code>KeyValue</code> data structure length
195    */
196   public static long getKeyValueDataStructureSize(int klength, int vlength, int tagsLength) {
197     if (tagsLength == 0) {
198       return KeyValue.KEYVALUE_INFRASTRUCTURE_SIZE + klength + vlength;
199     }
200     return KeyValue.KEYVALUE_WITH_TAGS_INFRASTRUCTURE_SIZE + klength + vlength + tagsLength;
201   }
202 
203   /**
204    * Computes the number of bytes that a <code>KeyValue</code> instance with the provided
205    * characteristics would take up in its underlying data structure for the key.
206    *
207    * @param rlength row length
208    * @param flength family length
209    * @param qlength qualifier length
210    *
211    * @return the key data structure length
212    */
213   public static long getKeyDataStructureSize(int rlength, int flength, int qlength) {
214     return KeyValue.KEY_INFRASTRUCTURE_SIZE + rlength + flength + qlength;
215   }
216 
217   /**
218    * Key type.
219    * Has space for other key types to be added later.  Cannot rely on
220    * enum ordinals . They change if item is removed or moved.  Do our own codes.
221    */
222   public static enum Type {
223     Minimum((byte)0),
224     Put((byte)4),
225 
226     Delete((byte)8),
227     DeleteFamilyVersion((byte)10),
228     DeleteColumn((byte)12),
229     DeleteFamily((byte)14),
230 
231     // Maximum is used when searching; you look from maximum on down.
232     Maximum((byte)255);
233 
234     private final byte code;
235 
236     Type(final byte c) {
237       this.code = c;
238     }
239 
240     public byte getCode() {
241       return this.code;
242     }
243 
244     /**
245      * Cannot rely on enum ordinals . They change if item is removed or moved.
246      * Do our own codes.
247      * @param b
248      * @return Type associated with passed code.
249      */
250     public static Type codeToType(final byte b) {
251       for (Type t : Type.values()) {
252         if (t.getCode() == b) {
253           return t;
254         }
255       }
256       throw new RuntimeException("Unknown code " + b);
257     }
258   }
259 
260   /**
261    * Lowest possible key.
262    * Makes a Key with highest possible Timestamp, empty row and column.  No
263    * key can be equal or lower than this one in memstore or in store file.
264    */
265   public static final KeyValue LOWESTKEY =
266     new KeyValue(HConstants.EMPTY_BYTE_ARRAY, HConstants.LATEST_TIMESTAMP);
267 
268   ////
269   // KeyValue core instance fields.
270   private byte [] bytes = null;  // an immutable byte array that contains the KV
271   private int offset = 0;  // offset into bytes buffer KV starts at
272   private int length = 0;  // length of the KV starting from offset.
273 
274   /**
275    * @return True if a delete type, a {@link KeyValue.Type#Delete} or
276    * a {KeyValue.Type#DeleteFamily} or a {@link KeyValue.Type#DeleteColumn}
277    * KeyValue type.
278    */
279   public static boolean isDelete(byte t) {
280     return Type.Delete.getCode() <= t && t <= Type.DeleteFamily.getCode();
281   }
282 
283   /** Here be dragons **/
284 
285   // used to achieve atomic operations in the memstore.
286   @Override
287   public long getMvccVersion() {
288     return mvcc;
289   }
290 
291   public void setMvccVersion(long mvccVersion){
292     this.mvcc = mvccVersion;
293   }
294 
295   // multi-version concurrency control version.  default value is 0, aka do not care.
296   private long mvcc = 0;  // this value is not part of a serialized KeyValue (not in HFiles)
297 
298   /** Dragon time over, return to normal business */
299 
300 
301   /** Writable Constructor -- DO NOT USE */
302   public KeyValue() {}
303 
304   /**
305    * Creates a KeyValue from the start of the specified byte array.
306    * Presumes <code>bytes</code> content is formatted as a KeyValue blob.
307    * @param bytes byte array
308    */
309   public KeyValue(final byte [] bytes) {
310     this(bytes, 0);
311   }
312 
313   /**
314    * Creates a KeyValue from the specified byte array and offset.
315    * Presumes <code>bytes</code> content starting at <code>offset</code> is
316    * formatted as a KeyValue blob.
317    * @param bytes byte array
318    * @param offset offset to start of KeyValue
319    */
320   public KeyValue(final byte [] bytes, final int offset) {
321     this(bytes, offset, getLength(bytes, offset));
322   }
323 
324   /**
325    * Creates a KeyValue from the specified byte array, starting at offset, and
326    * for length <code>length</code>.
327    * @param bytes byte array
328    * @param offset offset to start of the KeyValue
329    * @param length length of the KeyValue
330    */
331   public KeyValue(final byte [] bytes, final int offset, final int length) {
332     this.bytes = bytes;
333     this.offset = offset;
334     this.length = length;
335   }
336 
337   /**
338    * Creates a KeyValue from the specified byte array, starting at offset, and
339    * for length <code>length</code>.
340    *
341    * @param bytes  byte array
342    * @param offset offset to start of the KeyValue
343    * @param length length of the KeyValue
344    * @param ts
345    */
346   public KeyValue(final byte[] bytes, final int offset, final int length, long ts) {
347     this(bytes, offset, length, null, 0, 0, null, 0, 0, ts, Type.Maximum, null, 0, 0, null);
348   }
349 
350   /** Constructors that build a new backing byte array from fields */
351 
352   /**
353    * Constructs KeyValue structure filled with null value.
354    * Sets type to {@link KeyValue.Type#Maximum}
355    * @param row - row key (arbitrary byte array)
356    * @param timestamp
357    */
358   public KeyValue(final byte [] row, final long timestamp) {
359     this(row, null, null, timestamp, Type.Maximum, null);
360   }
361 
362   /**
363    * Constructs KeyValue structure filled with null value.
364    * @param row - row key (arbitrary byte array)
365    * @param timestamp
366    */
367   public KeyValue(final byte [] row, final long timestamp, Type type) {
368     this(row, null, null, timestamp, type, null);
369   }
370 
371   /**
372    * Constructs KeyValue structure filled with null value.
373    * Sets type to {@link KeyValue.Type#Maximum}
374    * @param row - row key (arbitrary byte array)
375    * @param family family name
376    * @param qualifier column qualifier
377    */
378   public KeyValue(final byte [] row, final byte [] family,
379       final byte [] qualifier) {
380     this(row, family, qualifier, HConstants.LATEST_TIMESTAMP, Type.Maximum);
381   }
382 
383   /**
384    * Constructs KeyValue structure filled with null value.
385    * @param row - row key (arbitrary byte array)
386    * @param family family name
387    * @param qualifier column qualifier
388    */
389   public KeyValue(final byte [] row, final byte [] family,
390       final byte [] qualifier, final byte [] value) {
391     this(row, family, qualifier, HConstants.LATEST_TIMESTAMP, Type.Put, value);
392   }
393 
394   /**
395    * Constructs KeyValue structure filled with specified values.
396    * @param row row key
397    * @param family family name
398    * @param qualifier column qualifier
399    * @param timestamp version timestamp
400    * @param type key type
401    * @throws IllegalArgumentException
402    */
403   public KeyValue(final byte[] row, final byte[] family,
404       final byte[] qualifier, final long timestamp, Type type) {
405     this(row, family, qualifier, timestamp, type, null);
406   }
407 
408   /**
409    * Constructs KeyValue structure filled with specified values.
410    * @param row row key
411    * @param family family name
412    * @param qualifier column qualifier
413    * @param timestamp version timestamp
414    * @param value column value
415    * @throws IllegalArgumentException
416    */
417   public KeyValue(final byte[] row, final byte[] family,
418       final byte[] qualifier, final long timestamp, final byte[] value) {
419     this(row, family, qualifier, timestamp, Type.Put, value);
420   }
421 
422   /**
423    * Constructs KeyValue structure filled with specified values.
424    * @param row row key
425    * @param family family name
426    * @param qualifier column qualifier
427    * @param timestamp version timestamp
428    * @param value column value
429    * @param tags tags
430    * @throws IllegalArgumentException
431    */
432   public KeyValue(final byte[] row, final byte[] family,
433       final byte[] qualifier, final long timestamp, final byte[] value,
434       final Tag[] tags) {
435     this(row, family, qualifier, timestamp, value, tags != null ? Arrays.asList(tags) : null);
436   }
437 
438   /**
439    * Constructs KeyValue structure filled with specified values.
440    * @param row row key
441    * @param family family name
442    * @param qualifier column qualifier
443    * @param timestamp version timestamp
444    * @param value column value
445    * @param tags tags non-empty list of tags or null
446    * @throws IllegalArgumentException
447    */
448   public KeyValue(final byte[] row, final byte[] family,
449       final byte[] qualifier, final long timestamp, final byte[] value,
450       final List<Tag> tags) {
451     this(row, 0, row==null ? 0 : row.length,
452       family, 0, family==null ? 0 : family.length,
453       qualifier, 0, qualifier==null ? 0 : qualifier.length,
454       timestamp, Type.Put,
455       value, 0, value==null ? 0 : value.length, tags);
456   }
457 
458   /**
459    * Constructs KeyValue structure filled with specified values.
460    * @param row row key
461    * @param family family name
462    * @param qualifier column qualifier
463    * @param timestamp version timestamp
464    * @param type key type
465    * @param value column value
466    * @throws IllegalArgumentException
467    */
468   public KeyValue(final byte[] row, final byte[] family,
469       final byte[] qualifier, final long timestamp, Type type,
470       final byte[] value) {
471     this(row, 0, len(row),   family, 0, len(family),   qualifier, 0, len(qualifier),
472         timestamp, type,   value, 0, len(value));
473   }
474 
475   /**
476    * Constructs KeyValue structure filled with specified values.
477    * <p>
478    * Column is split into two fields, family and qualifier.
479    * @param row row key
480    * @param family family name
481    * @param qualifier column qualifier
482    * @param timestamp version timestamp
483    * @param type key type
484    * @param value column value
485    * @throws IllegalArgumentException
486    */
487   public KeyValue(final byte[] row, final byte[] family,
488       final byte[] qualifier, final long timestamp, Type type,
489       final byte[] value, final List<Tag> tags) {
490     this(row, family, qualifier, 0, qualifier==null ? 0 : qualifier.length,
491         timestamp, type, value, 0, value==null ? 0 : value.length, tags);
492   }
493 
494   /**
495    * Constructs KeyValue structure filled with specified values.
496    * @param row row key
497    * @param family family name
498    * @param qualifier column qualifier
499    * @param timestamp version timestamp
500    * @param type key type
501    * @param value column value
502    * @throws IllegalArgumentException
503    */
504   public KeyValue(final byte[] row, final byte[] family,
505       final byte[] qualifier, final long timestamp, Type type,
506       final byte[] value, final byte[] tags) {
507     this(row, family, qualifier, 0, qualifier==null ? 0 : qualifier.length,
508         timestamp, type, value, 0, value==null ? 0 : value.length, tags);
509   }
510 
511   /**
512    * Constructs KeyValue structure filled with specified values.
513    * @param row row key
514    * @param family family name
515    * @param qualifier column qualifier
516    * @param qoffset qualifier offset
517    * @param qlength qualifier length
518    * @param timestamp version timestamp
519    * @param type key type
520    * @param value column value
521    * @param voffset value offset
522    * @param vlength value length
523    * @throws IllegalArgumentException
524    */
525   public KeyValue(byte [] row, byte [] family,
526       byte [] qualifier, int qoffset, int qlength, long timestamp, Type type,
527       byte [] value, int voffset, int vlength, List<Tag> tags) {
528     this(row, 0, row==null ? 0 : row.length,
529         family, 0, family==null ? 0 : family.length,
530         qualifier, qoffset, qlength, timestamp, type,
531         value, voffset, vlength, tags);
532   }
533 
534   /**
535    * @param row
536    * @param family
537    * @param qualifier
538    * @param qoffset
539    * @param qlength
540    * @param timestamp
541    * @param type
542    * @param value
543    * @param voffset
544    * @param vlength
545    * @param tags
546    */
547   public KeyValue(byte [] row, byte [] family,
548       byte [] qualifier, int qoffset, int qlength, long timestamp, Type type,
549       byte [] value, int voffset, int vlength, byte[] tags) {
550     this(row, 0, row==null ? 0 : row.length,
551         family, 0, family==null ? 0 : family.length,
552         qualifier, qoffset, qlength, timestamp, type,
553         value, voffset, vlength, tags, 0, tags==null ? 0 : tags.length);
554   }
555 
556   /**
557    * Constructs KeyValue structure filled with specified values.
558    * <p>
559    * Column is split into two fields, family and qualifier.
560    * @param row row key
561    * @throws IllegalArgumentException
562    */
563   public KeyValue(final byte [] row, final int roffset, final int rlength,
564       final byte [] family, final int foffset, final int flength,
565       final byte [] qualifier, final int qoffset, final int qlength,
566       final long timestamp, final Type type,
567       final byte [] value, final int voffset, final int vlength) {
568     this(row, roffset, rlength, family, foffset, flength, qualifier, qoffset,
569       qlength, timestamp, type, value, voffset, vlength, null);
570   }
571   
572   /**
573    * Constructs KeyValue structure filled with specified values. Uses the provided buffer as the
574    * data buffer.
575    * <p>
576    * Column is split into two fields, family and qualifier.
577    *
578    * @param buffer the bytes buffer to use
579    * @param boffset buffer offset
580    * @param row row key
581    * @param roffset row offset
582    * @param rlength row length
583    * @param family family name
584    * @param foffset family offset
585    * @param flength family length
586    * @param qualifier column qualifier
587    * @param qoffset qualifier offset
588    * @param qlength qualifier length
589    * @param timestamp version timestamp
590    * @param type key type
591    * @param value column value
592    * @param voffset value offset
593    * @param vlength value length
594    * @param tags non-empty list of tags or null
595    * @throws IllegalArgumentException an illegal value was passed or there is insufficient space
596    * remaining in the buffer
597    */
598   public KeyValue(byte [] buffer, final int boffset,
599       final byte [] row, final int roffset, final int rlength,
600       final byte [] family, final int foffset, final int flength,
601       final byte [] qualifier, final int qoffset, final int qlength,
602       final long timestamp, final Type type,
603       final byte [] value, final int voffset, final int vlength,
604       final Tag[] tags) {
605      this.bytes  = buffer;
606      this.length = writeByteArray(buffer, boffset,
607          row, roffset, rlength,
608          family, foffset, flength, qualifier, qoffset, qlength,
609         timestamp, type, value, voffset, vlength, tags);
610      this.offset = boffset;
611    }
612 
613   /**
614    * Constructs KeyValue structure filled with specified values.
615    * <p>
616    * Column is split into two fields, family and qualifier.
617    * @param row row key
618    * @param roffset row offset
619    * @param rlength row length
620    * @param family family name
621    * @param foffset family offset
622    * @param flength family length
623    * @param qualifier column qualifier
624    * @param qoffset qualifier offset
625    * @param qlength qualifier length
626    * @param timestamp version timestamp
627    * @param type key type
628    * @param value column value
629    * @param voffset value offset
630    * @param vlength value length
631    * @param tags tags
632    * @throws IllegalArgumentException
633    */
634   public KeyValue(final byte [] row, final int roffset, final int rlength,
635       final byte [] family, final int foffset, final int flength,
636       final byte [] qualifier, final int qoffset, final int qlength,
637       final long timestamp, final Type type,
638       final byte [] value, final int voffset, final int vlength,
639       final List<Tag> tags) {
640     this.bytes = createByteArray(row, roffset, rlength,
641         family, foffset, flength, qualifier, qoffset, qlength,
642         timestamp, type, value, voffset, vlength, tags);
643     this.length = bytes.length;
644     this.offset = 0;
645   }
646 
647   /**
648    * @param row
649    * @param roffset
650    * @param rlength
651    * @param family
652    * @param foffset
653    * @param flength
654    * @param qualifier
655    * @param qoffset
656    * @param qlength
657    * @param timestamp
658    * @param type
659    * @param value
660    * @param voffset
661    * @param vlength
662    * @param tags
663    */
664   public KeyValue(final byte [] row, final int roffset, final int rlength,
665       final byte [] family, final int foffset, final int flength,
666       final byte [] qualifier, final int qoffset, final int qlength,
667       final long timestamp, final Type type,
668       final byte [] value, final int voffset, final int vlength,
669       final byte[] tags, final int tagsOffset, final int tagsLength) {
670     this.bytes = createByteArray(row, roffset, rlength,
671         family, foffset, flength, qualifier, qoffset, qlength,
672         timestamp, type, value, voffset, vlength, tags, tagsOffset, tagsLength);
673     this.length = bytes.length;
674     this.offset = 0;
675   }
676 
677   /**
678    * Constructs an empty KeyValue structure, with specified sizes.
679    * This can be used to partially fill up KeyValues.
680    * <p>
681    * Column is split into two fields, family and qualifier.
682    * @param rlength row length
683    * @param flength family length
684    * @param qlength qualifier length
685    * @param timestamp version timestamp
686    * @param type key type
687    * @param vlength value length
688    * @throws IllegalArgumentException
689    */
690   public KeyValue(final int rlength,
691       final int flength,
692       final int qlength,
693       final long timestamp, final Type type,
694       final int vlength) {
695     this(rlength, flength, qlength, timestamp, type, vlength, 0);
696   }
697 
698   /**
699    * Constructs an empty KeyValue structure, with specified sizes.
700    * This can be used to partially fill up KeyValues.
701    * <p>
702    * Column is split into two fields, family and qualifier.
703    * @param rlength row length
704    * @param flength family length
705    * @param qlength qualifier length
706    * @param timestamp version timestamp
707    * @param type key type
708    * @param vlength value length
709    * @param tagsLength
710    * @throws IllegalArgumentException
711    */
712   public KeyValue(final int rlength,
713       final int flength,
714       final int qlength,
715       final long timestamp, final Type type,
716       final int vlength, final int tagsLength) {
717     this.bytes = createEmptyByteArray(rlength, flength, qlength, timestamp, type, vlength,
718         tagsLength);
719     this.length = bytes.length;
720     this.offset = 0;
721   }
722 
723 
724   public KeyValue(byte[] row, int roffset, int rlength,
725                   byte[] family, int foffset, int flength,
726                   ByteBuffer qualifier, long ts, Type type, ByteBuffer value, List<Tag> tags) {
727     this.bytes = createByteArray(row, roffset, rlength, family, foffset, flength,
728         qualifier, 0, qualifier == null ? 0 : qualifier.remaining(), ts, type,
729         value, 0, value == null ? 0 : value.remaining(), tags);
730     this.length = bytes.length;
731     this.offset = 0;
732   }
733 
734   public KeyValue(Cell c) {
735     this(c.getRowArray(), c.getRowOffset(), (int)c.getRowLength(),
736         c.getFamilyArray(), c.getFamilyOffset(), (int)c.getFamilyLength(), 
737         c.getQualifierArray(), c.getQualifierOffset(), (int) c.getQualifierLength(), 
738         c.getTimestamp(), Type.codeToType(c.getTypeByte()), c.getValueArray(), c.getValueOffset(), 
739         c.getValueLength(), c.getTagsArray(), c.getTagsOffset(), c.getTagsLengthUnsigned());
740   }
741 
742   /**
743    * Create an empty byte[] representing a KeyValue
744    * All lengths are preset and can be filled in later.
745    * @param rlength
746    * @param flength
747    * @param qlength
748    * @param timestamp
749    * @param type
750    * @param vlength
751    * @return The newly created byte array.
752    */
753   private static byte[] createEmptyByteArray(final int rlength, int flength,
754       int qlength, final long timestamp, final Type type, int vlength, int tagsLength) {
755     if (rlength > Short.MAX_VALUE) {
756       throw new IllegalArgumentException("Row > " + Short.MAX_VALUE);
757     }
758     if (flength > Byte.MAX_VALUE) {
759       throw new IllegalArgumentException("Family > " + Byte.MAX_VALUE);
760     }
761     // Qualifier length
762     if (qlength > Integer.MAX_VALUE - rlength - flength) {
763       throw new IllegalArgumentException("Qualifier > " + Integer.MAX_VALUE);
764     }
765     checkForTagsLength(tagsLength);
766     // Key length
767     long longkeylength = getKeyDataStructureSize(rlength, flength, qlength);
768     if (longkeylength > Integer.MAX_VALUE) {
769       throw new IllegalArgumentException("keylength " + longkeylength + " > " +
770         Integer.MAX_VALUE);
771     }
772     int keylength = (int)longkeylength;
773     // Value length
774     if (vlength > HConstants.MAXIMUM_VALUE_LENGTH) { // FindBugs INT_VACUOUS_COMPARISON
775       throw new IllegalArgumentException("Valuer > " +
776           HConstants.MAXIMUM_VALUE_LENGTH);
777     }
778 
779     // Allocate right-sized byte array.
780     byte[] bytes= new byte[(int) getKeyValueDataStructureSize(rlength, flength, qlength, vlength,
781         tagsLength)];
782     // Write the correct size markers
783     int pos = 0;
784     pos = Bytes.putInt(bytes, pos, keylength);
785     pos = Bytes.putInt(bytes, pos, vlength);
786     pos = Bytes.putShort(bytes, pos, (short)(rlength & 0x0000ffff));
787     pos += rlength;
788     pos = Bytes.putByte(bytes, pos, (byte)(flength & 0x0000ff));
789     pos += flength + qlength;
790     pos = Bytes.putLong(bytes, pos, timestamp);
791     pos = Bytes.putByte(bytes, pos, type.getCode());
792     pos += vlength;
793     if (tagsLength > 0) {
794       pos = Bytes.putAsShort(bytes, pos, tagsLength);
795     }
796     return bytes;
797   }
798 
799   /**
800    * Checks the parameters passed to a constructor.
801    *
802    * @param row row key
803    * @param rlength row length
804    * @param family family name
805    * @param flength family length
806    * @param qlength qualifier length
807    * @param vlength value length
808    *
809    * @throws IllegalArgumentException an illegal value was passed
810    */
811   private static void checkParameters(final byte [] row, final int rlength,
812       final byte [] family, int flength, int qlength, int vlength)
813           throws IllegalArgumentException {
814     if (rlength > Short.MAX_VALUE) {
815       throw new IllegalArgumentException("Row > " + Short.MAX_VALUE);
816     }
817     if (row == null) {
818       throw new IllegalArgumentException("Row is null");
819     }
820     // Family length
821     flength = family == null ? 0 : flength;
822     if (flength > Byte.MAX_VALUE) {
823       throw new IllegalArgumentException("Family > " + Byte.MAX_VALUE);
824     }
825     // Qualifier length
826     if (qlength > Integer.MAX_VALUE - rlength - flength) {
827       throw new IllegalArgumentException("Qualifier > " + Integer.MAX_VALUE);
828     }
829     // Key length
830     long longKeyLength = getKeyDataStructureSize(rlength, flength, qlength);
831     if (longKeyLength > Integer.MAX_VALUE) {
832       throw new IllegalArgumentException("keylength " + longKeyLength + " > " +
833           Integer.MAX_VALUE);
834     }
835     // Value length
836     if (vlength > HConstants.MAXIMUM_VALUE_LENGTH) { // FindBugs INT_VACUOUS_COMPARISON
837       throw new IllegalArgumentException("Value length " + vlength + " > " +
838           HConstants.MAXIMUM_VALUE_LENGTH);
839     }
840   }
841 
842   /**
843    * Write KeyValue format into the provided byte array.
844    *
845    * @param buffer the bytes buffer to use
846    * @param boffset buffer offset
847    * @param row row key
848    * @param roffset row offset
849    * @param rlength row length
850    * @param family family name
851    * @param foffset family offset
852    * @param flength family length
853    * @param qualifier column qualifier
854    * @param qoffset qualifier offset
855    * @param qlength qualifier length
856    * @param timestamp version timestamp
857    * @param type key type
858    * @param value column value
859    * @param voffset value offset
860    * @param vlength value length
861    *
862    * @return The number of useful bytes in the buffer.
863    *
864    * @throws IllegalArgumentException an illegal value was passed or there is insufficient space
865    * remaining in the buffer
866    */
867   private static int writeByteArray(byte [] buffer, final int boffset,
868       final byte [] row, final int roffset, final int rlength,
869       final byte [] family, final int foffset, int flength,
870       final byte [] qualifier, final int qoffset, int qlength,
871       final long timestamp, final Type type,
872       final byte [] value, final int voffset, int vlength, Tag[] tags) {
873 
874     checkParameters(row, rlength, family, flength, qlength, vlength);
875 
876     // Calculate length of tags area
877     int tagsLength = 0;
878     if (tags != null && tags.length > 0) {
879       for (Tag t: tags) {
880         tagsLength += t.getLength();
881       }
882     }
883     checkForTagsLength(tagsLength);
884     int keyLength = (int) getKeyDataStructureSize(rlength, flength, qlength);
885     int keyValueLength = (int) getKeyValueDataStructureSize(rlength, flength, qlength, vlength,
886         tagsLength);
887     if (keyValueLength > buffer.length - boffset) {
888       throw new IllegalArgumentException("Buffer size " + (buffer.length - boffset) + " < " +
889           keyValueLength);
890     }
891 
892     // Write key, value and key row length.
893     int pos = boffset;
894     pos = Bytes.putInt(buffer, pos, keyLength);
895     pos = Bytes.putInt(buffer, pos, vlength);
896     pos = Bytes.putShort(buffer, pos, (short)(rlength & 0x0000ffff));
897     pos = Bytes.putBytes(buffer, pos, row, roffset, rlength);
898     pos = Bytes.putByte(buffer, pos, (byte) (flength & 0x0000ff));
899     if (flength != 0) {
900       pos = Bytes.putBytes(buffer, pos, family, foffset, flength);
901     }
902     if (qlength != 0) {
903       pos = Bytes.putBytes(buffer, pos, qualifier, qoffset, qlength);
904     }
905     pos = Bytes.putLong(buffer, pos, timestamp);
906     pos = Bytes.putByte(buffer, pos, type.getCode());
907     if (value != null && value.length > 0) {
908       pos = Bytes.putBytes(buffer, pos, value, voffset, vlength);
909     }
910     // Write the number of tags. If it is 0 then it means there are no tags.
911     if (tagsLength > 0) {
912       pos = Bytes.putAsShort(buffer, pos, tagsLength);
913       for (Tag t : tags) {
914         pos = Bytes.putBytes(buffer, pos, t.getBuffer(), t.getOffset(), t.getLength());
915       }
916     }
917     return keyValueLength;
918   }
919 
920   private static void checkForTagsLength(int tagsLength) {
921     if (tagsLength > MAX_TAGS_LENGTH) {
922       throw new IllegalArgumentException("tagslength "+ tagsLength + " > " + MAX_TAGS_LENGTH);
923     }
924   }
925 
926   /**
927    * Write KeyValue format into a byte array.
928    * @param row row key
929    * @param roffset row offset
930    * @param rlength row length
931    * @param family family name
932    * @param foffset family offset
933    * @param flength family length
934    * @param qualifier column qualifier
935    * @param qoffset qualifier offset
936    * @param qlength qualifier length
937    * @param timestamp version timestamp
938    * @param type key type
939    * @param value column value
940    * @param voffset value offset
941    * @param vlength value length
942    * @return The newly created byte array.
943    */
944   private static byte [] createByteArray(final byte [] row, final int roffset,
945       final int rlength, final byte [] family, final int foffset, int flength,
946       final byte [] qualifier, final int qoffset, int qlength,
947       final long timestamp, final Type type,
948       final byte [] value, final int voffset, 
949       int vlength, byte[] tags, int tagsOffset, int tagsLength) {
950 
951     checkParameters(row, rlength, family, flength, qlength, vlength);
952     checkForTagsLength(tagsLength);
953     // Allocate right-sized byte array.
954     int keyLength = (int) getKeyDataStructureSize(rlength, flength, qlength);
955     byte [] bytes =
956         new byte[(int) getKeyValueDataStructureSize(rlength, flength, qlength, vlength, tagsLength)];
957     // Write key, value and key row length.
958     int pos = 0;
959     pos = Bytes.putInt(bytes, pos, keyLength);
960     pos = Bytes.putInt(bytes, pos, vlength);
961     pos = Bytes.putShort(bytes, pos, (short)(rlength & 0x0000ffff));
962     pos = Bytes.putBytes(bytes, pos, row, roffset, rlength);
963     pos = Bytes.putByte(bytes, pos, (byte)(flength & 0x0000ff));
964     if(flength != 0) {
965       pos = Bytes.putBytes(bytes, pos, family, foffset, flength);
966     }
967     if(qlength != 0) {
968       pos = Bytes.putBytes(bytes, pos, qualifier, qoffset, qlength);
969     }
970     pos = Bytes.putLong(bytes, pos, timestamp);
971     pos = Bytes.putByte(bytes, pos, type.getCode());
972     if (value != null && value.length > 0) {
973       pos = Bytes.putBytes(bytes, pos, value, voffset, vlength);
974     }
975     // Add the tags after the value part
976     if (tagsLength > 0) {
977       pos = Bytes.putAsShort(bytes, pos, tagsLength);
978       pos = Bytes.putBytes(bytes, pos, tags, tagsOffset, tagsLength);
979     }
980     return bytes;
981   }
982 
983   /**
984    * @param qualifier can be a ByteBuffer or a byte[], or null.
985    * @param value can be a ByteBuffer or a byte[], or null.
986    */
987   private static byte [] createByteArray(final byte [] row, final int roffset,
988       final int rlength, final byte [] family, final int foffset, int flength,
989       final Object qualifier, final int qoffset, int qlength,
990       final long timestamp, final Type type,
991       final Object value, final int voffset, int vlength, List<Tag> tags) {
992 
993     checkParameters(row, rlength, family, flength, qlength, vlength);
994 
995     // Calculate length of tags area
996     int tagsLength = 0;
997     if (tags != null && !tags.isEmpty()) {
998       for (Tag t : tags) {
999         tagsLength += t.getLength();
1000       }
1001     }
1002     checkForTagsLength(tagsLength);
1003     // Allocate right-sized byte array.
1004     int keyLength = (int) getKeyDataStructureSize(rlength, flength, qlength);
1005     byte[] bytes = new byte[(int) getKeyValueDataStructureSize(rlength, flength, qlength, vlength,
1006         tagsLength)];
1007 
1008     // Write key, value and key row length.
1009     int pos = 0;
1010     pos = Bytes.putInt(bytes, pos, keyLength);
1011 
1012     pos = Bytes.putInt(bytes, pos, vlength);
1013     pos = Bytes.putShort(bytes, pos, (short)(rlength & 0x0000ffff));
1014     pos = Bytes.putBytes(bytes, pos, row, roffset, rlength);
1015     pos = Bytes.putByte(bytes, pos, (byte)(flength & 0x0000ff));
1016     if(flength != 0) {
1017       pos = Bytes.putBytes(bytes, pos, family, foffset, flength);
1018     }
1019     if (qlength > 0) {
1020       if (qualifier instanceof ByteBuffer) {
1021         pos = Bytes.putByteBuffer(bytes, pos, (ByteBuffer) qualifier);
1022       } else {
1023         pos = Bytes.putBytes(bytes, pos, (byte[]) qualifier, qoffset, qlength);
1024       }
1025     }
1026     pos = Bytes.putLong(bytes, pos, timestamp);
1027     pos = Bytes.putByte(bytes, pos, type.getCode());
1028     if (vlength > 0) {
1029       if (value instanceof ByteBuffer) {
1030         pos = Bytes.putByteBuffer(bytes, pos, (ByteBuffer) value);
1031       } else {
1032         pos = Bytes.putBytes(bytes, pos, (byte[]) value, voffset, vlength);
1033       }
1034     }
1035     // Add the tags after the value part
1036     if (tagsLength > 0) {
1037       pos = Bytes.putAsShort(bytes, pos, tagsLength);
1038       for (Tag t : tags) {
1039         pos = Bytes.putBytes(bytes, pos, t.getBuffer(), t.getOffset(), t.getLength());
1040       }
1041     }
1042     return bytes;
1043   }
1044 
1045   /**
1046    * Needed doing 'contains' on List.  Only compares the key portion, not the value.
1047    */
1048   @Override
1049   public boolean equals(Object other) {
1050     if (!(other instanceof Cell)) {
1051       return false;
1052     }
1053     return CellComparator.equals(this, (Cell)other);
1054   }
1055 
1056   @Override
1057   public int hashCode() {
1058     byte[] b = getBuffer();
1059     int start = getOffset(), end = getOffset() + getLength();
1060     int h = b[start++];
1061     for (int i = start; i < end; i++) {
1062       h = (h * 13) ^ b[i];
1063     }
1064     return h;
1065   }
1066 
1067   //---------------------------------------------------------------------------
1068   //
1069   //  KeyValue cloning
1070   //
1071   //---------------------------------------------------------------------------
1072 
1073   /**
1074    * Clones a KeyValue.  This creates a copy, re-allocating the buffer.
1075    * @return Fully copied clone of this KeyValue
1076    * @throws CloneNotSupportedException
1077    */
1078   @Override
1079   public KeyValue clone() throws CloneNotSupportedException {
1080     super.clone();
1081     byte [] b = new byte[this.length];
1082     System.arraycopy(this.bytes, this.offset, b, 0, this.length);
1083     KeyValue ret = new KeyValue(b, 0, b.length);
1084     // Important to clone the memstoreTS as well - otherwise memstore's
1085     // update-in-place methods (eg increment) will end up creating
1086     // new entries
1087     ret.setMvccVersion(mvcc);
1088     return ret;
1089   }
1090 
1091   /**
1092    * Creates a shallow copy of this KeyValue, reusing the data byte buffer.
1093    * http://en.wikipedia.org/wiki/Object_copy
1094    * @return Shallow copy of this KeyValue
1095    */
1096   public KeyValue shallowCopy() {
1097     KeyValue shallowCopy = new KeyValue(this.bytes, this.offset, this.length);
1098     shallowCopy.setMvccVersion(this.mvcc);
1099     return shallowCopy;
1100   }
1101 
1102   //---------------------------------------------------------------------------
1103   //
1104   //  String representation
1105   //
1106   //---------------------------------------------------------------------------
1107 
1108   public String toString() {
1109     if (this.bytes == null || this.bytes.length == 0) {
1110       return "empty";
1111     }
1112     return keyToString(this.bytes, this.offset + ROW_OFFSET, getKeyLength()) +
1113       "/vlen=" + getValueLength() + "/mvcc=" + mvcc;
1114   }
1115 
1116   /**
1117    * @param k Key portion of a KeyValue.
1118    * @return Key as a String, empty string if k is null. 
1119    */
1120   public static String keyToString(final byte [] k) {
1121     if (k == null) { 
1122       return "";
1123     }
1124     return keyToString(k, 0, k.length);
1125   }
1126 
1127   /**
1128    * Produces a string map for this key/value pair. Useful for programmatic use
1129    * and manipulation of the data stored in an HLogKey, for example, printing
1130    * as JSON. Values are left out due to their tendency to be large. If needed,
1131    * they can be added manually.
1132    *
1133    * @return the Map<String,?> containing data from this key
1134    */
1135   public Map<String, Object> toStringMap() {
1136     Map<String, Object> stringMap = new HashMap<String, Object>();
1137     stringMap.put("row", Bytes.toStringBinary(getRow()));
1138     stringMap.put("family", Bytes.toStringBinary(getFamily()));
1139     stringMap.put("qualifier", Bytes.toStringBinary(getQualifier()));
1140     stringMap.put("timestamp", getTimestamp());
1141     stringMap.put("vlen", getValueLength());
1142     List<Tag> tags = getTags();
1143     if (tags != null) {
1144       List<String> tagsString = new ArrayList<String>();
1145       for (Tag t : tags) {
1146         tagsString.add((t.getType()) + ":" +Bytes.toStringBinary(t.getValue()));
1147       }
1148       stringMap.put("tag", tagsString);
1149     }
1150     return stringMap;
1151   }
1152 
1153   /**
1154    * Use for logging.
1155    * @param b Key portion of a KeyValue.
1156    * @param o Offset to start of key
1157    * @param l Length of key.
1158    * @return Key as a String.
1159    */
1160   public static String keyToString(final byte [] b, final int o, final int l) {
1161     if (b == null) return "";
1162     int rowlength = Bytes.toShort(b, o);
1163     String row = Bytes.toStringBinary(b, o + Bytes.SIZEOF_SHORT, rowlength);
1164     int columnoffset = o + Bytes.SIZEOF_SHORT + 1 + rowlength;
1165     int familylength = b[columnoffset - 1];
1166     int columnlength = l - ((columnoffset - o) + TIMESTAMP_TYPE_SIZE);
1167     String family = familylength == 0? "":
1168       Bytes.toStringBinary(b, columnoffset, familylength);
1169     String qualifier = columnlength == 0? "":
1170       Bytes.toStringBinary(b, columnoffset + familylength,
1171       columnlength - familylength);
1172     long timestamp = Bytes.toLong(b, o + (l - TIMESTAMP_TYPE_SIZE));
1173     String timestampStr = humanReadableTimestamp(timestamp);
1174     byte type = b[o + l - 1];
1175     return row + "/" + family +
1176       (family != null && family.length() > 0? ":" :"") +
1177       qualifier + "/" + timestampStr + "/" + Type.codeToType(type);
1178   }
1179 
1180   public static String humanReadableTimestamp(final long timestamp) {
1181     if (timestamp == HConstants.LATEST_TIMESTAMP) {
1182       return "LATEST_TIMESTAMP";
1183     }
1184     if (timestamp == HConstants.OLDEST_TIMESTAMP) {
1185       return "OLDEST_TIMESTAMP";
1186     }
1187     return String.valueOf(timestamp);
1188   }
1189 
1190   //---------------------------------------------------------------------------
1191   //
1192   //  Public Member Accessors
1193   //
1194   //---------------------------------------------------------------------------
1195 
1196   /**
1197    * @return The byte array backing this KeyValue.
1198    * @deprecated Since 0.98.0.  Use Cell Interface instead.  Do not presume single backing buffer.
1199    */
1200   @Deprecated
1201   public byte [] getBuffer() {
1202     return this.bytes;
1203   }
1204 
1205   /**
1206    * @return Offset into {@link #getBuffer()} at which this KeyValue starts.
1207    */
1208   public int getOffset() {
1209     return this.offset;
1210   }
1211 
1212   /**
1213    * @return Length of bytes this KeyValue occupies in {@link #getBuffer()}.
1214    */
1215   public int getLength() {
1216     return length;
1217   }
1218 
1219   //---------------------------------------------------------------------------
1220   //
1221   //  Length and Offset Calculators
1222   //
1223   //---------------------------------------------------------------------------
1224 
1225   /**
1226    * Determines the total length of the KeyValue stored in the specified
1227    * byte array and offset.  Includes all headers.
1228    * @param bytes byte array
1229    * @param offset offset to start of the KeyValue
1230    * @return length of entire KeyValue, in bytes
1231    */
1232   private static int getLength(byte [] bytes, int offset) {
1233     int klength = ROW_OFFSET + Bytes.toInt(bytes, offset);
1234     int vlength = Bytes.toInt(bytes, offset + Bytes.SIZEOF_INT);
1235     return klength + vlength;
1236   }
1237 
1238   /**
1239    * @return Key offset in backing buffer..
1240    */
1241   public int getKeyOffset() {
1242     return this.offset + ROW_OFFSET;
1243   }
1244 
1245   public String getKeyString() {
1246     return Bytes.toStringBinary(getBuffer(), getKeyOffset(), getKeyLength());
1247   }
1248 
1249   /**
1250    * @return Length of key portion.
1251    */
1252   public int getKeyLength() {
1253     return Bytes.toInt(this.bytes, this.offset);
1254   }
1255 
1256   /**
1257    * @return the backing array of the entire KeyValue (all KeyValue fields are in a single array)
1258    */
1259   @Override
1260   public byte[] getValueArray() {
1261     return bytes;
1262   }
1263 
1264   /**
1265    * @return the value offset
1266    */
1267   @Override
1268   public int getValueOffset() {
1269     int voffset = getKeyOffset() + getKeyLength();
1270     return voffset;
1271   }
1272 
1273   /**
1274    * @return Value length
1275    */
1276   @Override
1277   public int getValueLength() {
1278     int vlength = Bytes.toInt(this.bytes, this.offset + Bytes.SIZEOF_INT);
1279     return vlength;
1280   }
1281 
1282   /**
1283    * @return the backing array of the entire KeyValue (all KeyValue fields are in a single array)
1284    */
1285   @Override
1286   public byte[] getRowArray() {
1287     return bytes;
1288   }
1289 
1290   /**
1291    * @return Row offset
1292    */
1293   @Override
1294   public int getRowOffset() {
1295     return getKeyOffset() + Bytes.SIZEOF_SHORT;
1296   }
1297 
1298   /**
1299    * @return Row length
1300    */
1301   @Override
1302   public short getRowLength() {
1303     return Bytes.toShort(this.bytes, getKeyOffset());
1304   }
1305 
1306   /**
1307    * @return the backing array of the entire KeyValue (all KeyValue fields are in a single array)
1308    */
1309   @Override
1310   public byte[] getFamilyArray() {
1311     return bytes;
1312   }
1313 
1314   /**
1315    * @return Family offset
1316    */
1317   @Override
1318   public int getFamilyOffset() {
1319     return getFamilyOffset(getRowLength());
1320   }
1321 
1322   /**
1323    * @return Family offset
1324    */
1325   private int getFamilyOffset(int rlength) {
1326     return this.offset + ROW_OFFSET + Bytes.SIZEOF_SHORT + rlength + Bytes.SIZEOF_BYTE;
1327   }
1328 
1329   /**
1330    * @return Family length
1331    */
1332   @Override
1333   public byte getFamilyLength() {
1334     return getFamilyLength(getFamilyOffset());
1335   }
1336 
1337   /**
1338    * @return Family length
1339    */
1340   public byte getFamilyLength(int foffset) {
1341     return this.bytes[foffset-1];
1342   }
1343 
1344   /**
1345    * @return the backing array of the entire KeyValue (all KeyValue fields are in a single array)
1346    */
1347   @Override
1348   public byte[] getQualifierArray() {
1349     return bytes;
1350   }
1351 
1352   /**
1353    * @return Qualifier offset
1354    */
1355   @Override
1356   public int getQualifierOffset() {
1357     return getQualifierOffset(getFamilyOffset());
1358   }
1359 
1360   /**
1361    * @return Qualifier offset
1362    */
1363   private int getQualifierOffset(int foffset) {
1364     return foffset + getFamilyLength(foffset);
1365   }
1366 
1367   /**
1368    * @return Qualifier length
1369    */
1370   @Override
1371   public int getQualifierLength() {
1372     return getQualifierLength(getRowLength(),getFamilyLength());
1373   }
1374 
1375   /**
1376    * @return Qualifier length
1377    */
1378   private int getQualifierLength(int rlength, int flength) {
1379     return getKeyLength() - (int) getKeyDataStructureSize(rlength, flength, 0);
1380   }
1381 
1382   /**
1383    * @return Column (family + qualifier) length
1384    */
1385   private int getTotalColumnLength(int rlength, int foffset) {
1386     int flength = getFamilyLength(foffset);
1387     int qlength = getQualifierLength(rlength,flength);
1388     return flength + qlength;
1389   }
1390 
1391   /**
1392    * @return Timestamp offset
1393    */
1394   public int getTimestampOffset() {
1395     return getTimestampOffset(getKeyLength());
1396   }
1397 
1398   /**
1399    * @param keylength Pass if you have it to save on a int creation.
1400    * @return Timestamp offset
1401    */
1402   private int getTimestampOffset(final int keylength) {
1403     return getKeyOffset() + keylength - TIMESTAMP_TYPE_SIZE;
1404   }
1405 
1406   /**
1407    * @return True if this KeyValue has a LATEST_TIMESTAMP timestamp.
1408    */
1409   public boolean isLatestTimestamp() {
1410     return Bytes.equals(getBuffer(), getTimestampOffset(), Bytes.SIZEOF_LONG,
1411       HConstants.LATEST_TIMESTAMP_BYTES, 0, Bytes.SIZEOF_LONG);
1412   }
1413 
1414   /**
1415    * @param now Time to set into <code>this</code> IFF timestamp ==
1416    * {@link HConstants#LATEST_TIMESTAMP} (else, its a noop).
1417    * @return True is we modified this.
1418    */
1419   public boolean updateLatestStamp(final byte [] now) {
1420     if (this.isLatestTimestamp()) {
1421       int tsOffset = getTimestampOffset();
1422       System.arraycopy(now, 0, this.bytes, tsOffset, Bytes.SIZEOF_LONG);
1423       // clear cache or else getTimestamp() possibly returns an old value
1424       return true;
1425     }
1426     return false;
1427   }
1428 
1429   //---------------------------------------------------------------------------
1430   //
1431   //  Methods that return copies of fields
1432   //
1433   //---------------------------------------------------------------------------
1434 
1435   /**
1436    * Do not use unless you have to.  Used internally for compacting and testing.
1437    *
1438    * Use {@link #getRow()}, {@link #getFamily()}, {@link #getQualifier()}, and
1439    * {@link #getValue()} if accessing a KeyValue client-side.
1440    * @return Copy of the key portion only.
1441    */
1442   public byte [] getKey() {
1443     int keylength = getKeyLength();
1444     byte [] key = new byte[keylength];
1445     System.arraycopy(getBuffer(), getKeyOffset(), key, 0, keylength);
1446     return key;
1447   }
1448 
1449   /**
1450    * Returns value in a new byte array.
1451    * Primarily for use client-side. If server-side, use
1452    * {@link #getBuffer()} with appropriate offsets and lengths instead to
1453    * save on allocations.
1454    * @return Value in a new byte array.
1455    */
1456   @Deprecated // use CellUtil.getValueArray()
1457   public byte [] getValue() {
1458     return CellUtil.cloneValue(this);
1459   }
1460 
1461   /**
1462    * Primarily for use client-side.  Returns the row of this KeyValue in a new
1463    * byte array.<p>
1464    *
1465    * If server-side, use {@link #getBuffer()} with appropriate offsets and
1466    * lengths instead.
1467    * @return Row in a new byte array.
1468    */
1469   @Deprecated // use CellUtil.getRowArray()
1470   public byte [] getRow() {
1471     return CellUtil.cloneRow(this);
1472   }
1473 
1474   /**
1475    *
1476    * @return Timestamp
1477    */
1478   @Override
1479   public long getTimestamp() {
1480     return getTimestamp(getKeyLength());
1481   }
1482 
1483   /**
1484    * @param keylength Pass if you have it to save on a int creation.
1485    * @return Timestamp
1486    */
1487   long getTimestamp(final int keylength) {
1488     int tsOffset = getTimestampOffset(keylength);
1489     return Bytes.toLong(this.bytes, tsOffset);
1490   }
1491 
1492   /**
1493    * @return Type of this KeyValue.
1494    */
1495   @Deprecated
1496   public byte getType() {
1497     return getTypeByte();
1498   }
1499 
1500   /**
1501    * @return KeyValue.TYPE byte representation
1502    */
1503   @Override
1504   public byte getTypeByte() {
1505     return this.bytes[this.offset + getKeyLength() - 1 + ROW_OFFSET];
1506   }
1507 
1508   /**
1509    * @return True if a delete type, a {@link KeyValue.Type#Delete} or
1510    * a {KeyValue.Type#DeleteFamily} or a {@link KeyValue.Type#DeleteColumn}
1511    * KeyValue type.
1512    */
1513   @Deprecated // use CellUtil#isDelete
1514   public boolean isDelete() {
1515     return KeyValue.isDelete(getType());
1516   }
1517 
1518   /**
1519    * @return True if this KV is a {@link KeyValue.Type#Delete} type.
1520    */
1521   public boolean isDeleteType() {
1522     // TODO: Fix this method name vis-a-vis isDelete!
1523     return getTypeByte() == Type.Delete.getCode();
1524   }
1525 
1526   /**
1527    * @return True if this KV is a delete family type.
1528    */
1529   public boolean isDeleteFamily() {
1530     return getTypeByte() == Type.DeleteFamily.getCode();
1531   }
1532 
1533   /**
1534    * @return True if this KV is a delete family-version type.
1535    */
1536   public boolean isDeleteFamilyVersion() {
1537     return getTypeByte() == Type.DeleteFamilyVersion.getCode();
1538   }
1539 
1540   /**
1541    *
1542    * @return True if this KV is a delete family or column type.
1543    */
1544   public boolean isDeleteColumnOrFamily() {
1545     int t = getTypeByte();
1546     return t == Type.DeleteColumn.getCode() || t == Type.DeleteFamily.getCode();
1547   }
1548 
1549   /**
1550    * Primarily for use client-side.  Returns the family of this KeyValue in a
1551    * new byte array.<p>
1552    *
1553    * If server-side, use {@link #getBuffer()} with appropriate offsets and
1554    * lengths instead.
1555    * @return Returns family. Makes a copy.
1556    */
1557   @Deprecated // use CellUtil.getFamilyArray
1558   public byte [] getFamily() {
1559     return CellUtil.cloneFamily(this);
1560   }
1561 
1562   /**
1563    * Primarily for use client-side.  Returns the column qualifier of this
1564    * KeyValue in a new byte array.<p>
1565    *
1566    * If server-side, use {@link #getBuffer()} with appropriate offsets and
1567    * lengths instead.
1568    * Use {@link #getBuffer()} with appropriate offsets and lengths instead.
1569    * @return Returns qualifier. Makes a copy.
1570    */
1571   @Deprecated // use CellUtil.getQualifierArray
1572   public byte [] getQualifier() {
1573     return CellUtil.cloneQualifier(this);
1574   }
1575 
1576   /**
1577    * This returns the offset where the tag actually starts.
1578    */
1579   @Override
1580   public int getTagsOffset() {
1581     int tagsLen = getTagsLengthUnsigned();
1582     if (tagsLen == 0) {
1583       return this.offset + this.length;
1584     }
1585     return this.offset + this.length - tagsLen;
1586   }
1587 
1588   /**
1589    * This returns the total length of the tag bytes
1590    */
1591   @Override
1592   @Deprecated
1593   public int getTagsLengthUnsigned() {
1594     int tagsLen = this.length - (getKeyLength() + getValueLength() + KEYVALUE_INFRASTRUCTURE_SIZE);
1595     if (tagsLen > 0) {
1596       // There are some Tag bytes in the byte[]. So reduce 2 bytes which is added to denote the tags
1597       // length
1598       tagsLen -= TAGS_LENGTH_SIZE;
1599     }
1600     return tagsLen;
1601   }
1602 
1603   @Override
1604   @Deprecated
1605   public short getTagsLength() {
1606     return (short) getTagsLengthUnsigned();
1607   }
1608 
1609   /**
1610    * Returns any tags embedded in the KeyValue.  Used in testcases.
1611    * @return The tags
1612    */
1613   public List<Tag> getTags() {
1614     int tagsLength = getTagsLengthUnsigned();
1615     if (tagsLength == 0) {
1616       return EMPTY_ARRAY_LIST;
1617     }
1618     return Tag.asList(getBuffer(), getTagsOffset(), tagsLength);
1619   }
1620 
1621   /**
1622    * @return the backing array of the entire KeyValue (all KeyValue fields are in a single array)
1623    */
1624   @Override
1625   public byte[] getTagsArray() {
1626     return bytes;
1627   }
1628 
1629   //---------------------------------------------------------------------------
1630   //
1631   //  Compare specified fields against those contained in this KeyValue
1632   //
1633   //---------------------------------------------------------------------------
1634 
1635   /**
1636    * @param family
1637    * @return True if matching families.
1638    */
1639   public boolean matchingFamily(final byte [] family) {
1640     if (this.length == 0 || this.bytes.length == 0) {
1641       return false;
1642     }
1643     return Bytes.equals(family, 0, family.length,
1644         this.bytes, getFamilyOffset(), getFamilyLength());
1645   }
1646 
1647   /**
1648    * @param qualifier
1649    * @return True if matching qualifiers.
1650    */
1651   public boolean matchingQualifier(final byte [] qualifier) {
1652     return matchingQualifier(qualifier, 0, qualifier.length);
1653   }
1654 
1655   public boolean matchingQualifier(final byte [] qualifier, int offset, int length) {
1656     return Bytes.equals(qualifier, offset, length,
1657         this.bytes, getQualifierOffset(), getQualifierLength());
1658   }
1659 
1660   public boolean matchingQualifier(final KeyValue other) {
1661     return matchingQualifier(other.getBuffer(), other.getQualifierOffset(),
1662         other.getQualifierLength());
1663   }
1664 
1665   public boolean matchingRow(final byte [] row) {
1666     return matchingRow(row, 0, row.length);
1667   }
1668 
1669   public boolean matchingRow(final byte[] row, int offset, int length) {
1670     return Bytes.equals(row, offset, length,
1671         this.bytes, getRowOffset(), getRowLength());
1672   }
1673 
1674   public boolean matchingRow(KeyValue other) {
1675     return matchingRow(other.getBuffer(), other.getRowOffset(),
1676         other.getRowLength());
1677   }
1678 
1679   /**
1680    *
1681    * @param family column family
1682    * @param qualifier column qualifier
1683    * @return True if column matches
1684    */
1685   public boolean matchingColumn(final byte[] family, final byte[] qualifier) {
1686     return matchingColumn(family, 0, len(family), qualifier, 0, len(qualifier));
1687   }
1688 
1689   /**
1690    * Checks if column matches.
1691    *
1692    * @param family family name
1693    * @param foffset family offset
1694    * @param flength family length
1695    * @param qualifier column qualifier
1696    * @param qoffset qualifier offset
1697    * @param qlength qualifier length
1698    *
1699    * @return True if column matches
1700    */
1701   public boolean matchingColumn(final byte [] family, final int foffset, final int flength,
1702       final byte [] qualifier, final int qoffset, final int qlength) {
1703     int rl = getRowLength();
1704     int o = getFamilyOffset(rl);
1705     int fl = getFamilyLength(o);
1706     if (!Bytes.equals(family, foffset, flength, this.bytes, o, fl)) {
1707       return false;
1708     }
1709 
1710     int ql = getQualifierLength(rl, fl);
1711     if (qualifier == null || qlength == 0) {
1712       return (ql == 0);
1713     }
1714     return Bytes.equals(qualifier, qoffset, qlength, this.bytes, o + fl, ql);
1715   }
1716 
1717   /**
1718    * Creates a new KeyValue that only contains the key portion (the value is
1719    * set to be null).
1720    *
1721    * TODO only used by KeyOnlyFilter -- move there.
1722    * @param lenAsVal replace value with the actual value length (false=empty)
1723    */
1724   public KeyValue createKeyOnly(boolean lenAsVal) {
1725     // KV format:  <keylen:4><valuelen:4><key:keylen><value:valuelen>
1726     // Rebuild as: <keylen:4><0:4><key:keylen>
1727     int dataLen = lenAsVal? Bytes.SIZEOF_INT : 0;
1728     byte [] newBuffer = new byte[getKeyLength() + ROW_OFFSET + dataLen];
1729     System.arraycopy(this.bytes, this.offset, newBuffer, 0,
1730         Math.min(newBuffer.length,this.length));
1731     Bytes.putInt(newBuffer, Bytes.SIZEOF_INT, dataLen);
1732     if (lenAsVal) {
1733       Bytes.putInt(newBuffer, newBuffer.length - dataLen, this.getValueLength());
1734     }
1735     return new KeyValue(newBuffer);
1736   }
1737 
1738   /**
1739    * Splits a column in {@code family:qualifier} form into separate byte arrays. An empty qualifier
1740    * (ie, {@code fam:}) is parsed as <code>{ fam, EMPTY_BYTE_ARRAY }</code> while no delimiter (ie,
1741    * {@code fam}) is parsed as an array of one element, <code>{ fam }</code>.
1742    * <p>
1743    * Don't forget, HBase DOES support empty qualifiers. (see HBASE-9549)
1744    * </p>
1745    * <p>
1746    * Not recommend to be used as this is old-style API.
1747    * </p>
1748    * @param c The column.
1749    * @return The parsed column.
1750    */
1751   public static byte [][] parseColumn(byte [] c) {
1752     final int index = getDelimiter(c, 0, c.length, COLUMN_FAMILY_DELIMITER);
1753     if (index == -1) {
1754       // If no delimiter, return array of size 1
1755       return new byte [][] { c };
1756     } else if(index == c.length - 1) {
1757       // family with empty qualifier, return array size 2
1758       byte [] family = new byte[c.length-1];
1759       System.arraycopy(c, 0, family, 0, family.length);
1760       return new byte [][] { family, HConstants.EMPTY_BYTE_ARRAY};
1761     }
1762     // Family and column, return array size 2
1763     final byte [][] result = new byte [2][];
1764     result[0] = new byte [index];
1765     System.arraycopy(c, 0, result[0], 0, index);
1766     final int len = c.length - (index + 1);
1767     result[1] = new byte[len];
1768     System.arraycopy(c, index + 1 /* Skip delimiter */, result[1], 0, len);
1769     return result;
1770   }
1771 
1772   /**
1773    * Makes a column in family:qualifier form from separate byte arrays.
1774    * <p>
1775    * Not recommended for usage as this is old-style API.
1776    * @param family
1777    * @param qualifier
1778    * @return family:qualifier
1779    */
1780   public static byte [] makeColumn(byte [] family, byte [] qualifier) {
1781     return Bytes.add(family, COLUMN_FAMILY_DELIM_ARRAY, qualifier);
1782   }
1783 
1784   /**
1785    * @param b
1786    * @param delimiter
1787    * @return Index of delimiter having started from start of <code>b</code>
1788    * moving rightward.
1789    */
1790   public static int getDelimiter(final byte [] b, int offset, final int length,
1791       final int delimiter) {
1792     if (b == null) {
1793       throw new IllegalArgumentException("Passed buffer is null");
1794     }
1795     int result = -1;
1796     for (int i = offset; i < length + offset; i++) {
1797       if (b[i] == delimiter) {
1798         result = i;
1799         break;
1800       }
1801     }
1802     return result;
1803   }
1804 
1805   /**
1806    * Find index of passed delimiter walking from end of buffer backwards.
1807    * @param b
1808    * @param delimiter
1809    * @return Index of delimiter
1810    */
1811   public static int getDelimiterInReverse(final byte [] b, final int offset,
1812       final int length, final int delimiter) {
1813     if (b == null) {
1814       throw new IllegalArgumentException("Passed buffer is null");
1815     }
1816     int result = -1;
1817     for (int i = (offset + length) - 1; i >= offset; i--) {
1818       if (b[i] == delimiter) {
1819         result = i;
1820         break;
1821       }
1822     }
1823     return result;
1824   }
1825 
1826   /**
1827    * A {@link KVComparator} for <code>hbase:meta</code> catalog table
1828    * {@link KeyValue}s.
1829    */
1830   public static class MetaComparator extends KVComparator {
1831     /**
1832      * Compare key portion of a {@link KeyValue} for keys in <code>hbase:meta</code>
1833      * table.
1834      */
1835     @Override
1836     public int compareRows(byte [] left, int loffset, int llength,
1837         byte [] right, int roffset, int rlength) {
1838       int leftDelimiter = getDelimiter(left, loffset, llength,
1839           HConstants.DELIMITER);
1840       int rightDelimiter = getDelimiter(right, roffset, rlength,
1841           HConstants.DELIMITER);
1842       // Compare up to the delimiter
1843       int lpart = (leftDelimiter < 0 ? llength :leftDelimiter - loffset);
1844       int rpart = (rightDelimiter < 0 ? rlength :rightDelimiter - roffset);
1845       int result = Bytes.compareTo(left, loffset, lpart, right, roffset, rpart);
1846       if (result != 0) {
1847         return result;
1848       } else {
1849         if (leftDelimiter < 0 && rightDelimiter >= 0) {
1850           return -1;
1851         } else if (rightDelimiter < 0 && leftDelimiter >= 0) {
1852           return 1;
1853         } else if (leftDelimiter < 0 && rightDelimiter < 0) {
1854           return 0;
1855         }
1856       }
1857       // Compare middle bit of the row.
1858       // Move past delimiter
1859       leftDelimiter++;
1860       rightDelimiter++;
1861       int leftFarDelimiter = getDelimiterInReverse(left, leftDelimiter,
1862           llength - (leftDelimiter - loffset), HConstants.DELIMITER);
1863       int rightFarDelimiter = getDelimiterInReverse(right,
1864           rightDelimiter, rlength - (rightDelimiter - roffset),
1865           HConstants.DELIMITER);
1866       // Now compare middlesection of row.
1867       lpart = (leftFarDelimiter < 0 ? llength + loffset: leftFarDelimiter) - leftDelimiter;
1868       rpart = (rightFarDelimiter < 0 ? rlength + roffset: rightFarDelimiter)- rightDelimiter;
1869       result = super.compareRows(left, leftDelimiter, lpart, right, rightDelimiter, rpart);
1870       if (result != 0) {
1871         return result;
1872       }  else {
1873         if (leftDelimiter < 0 && rightDelimiter >= 0) {
1874           return -1;
1875         } else if (rightDelimiter < 0 && leftDelimiter >= 0) {
1876           return 1;
1877         } else if (leftDelimiter < 0 && rightDelimiter < 0) {
1878           return 0;
1879         }
1880       }
1881       // Compare last part of row, the rowid.
1882       leftFarDelimiter++;
1883       rightFarDelimiter++;
1884       result = Bytes.compareTo(left, leftFarDelimiter, llength - (leftFarDelimiter - loffset),
1885           right, rightFarDelimiter, rlength - (rightFarDelimiter - roffset));
1886       return result;
1887     }
1888 
1889     /**
1890      * Don't do any fancy Block Index splitting tricks.
1891      */
1892     @Override
1893     public byte[] getShortMidpointKey(final byte[] leftKey, final byte[] rightKey) {
1894       return Arrays.copyOf(rightKey, rightKey.length);
1895     }
1896 
1897     /**
1898      * The HFileV2 file format's trailer contains this class name.  We reinterpret this and
1899      * instantiate the appropriate comparator.
1900      * TODO: With V3 consider removing this.
1901      * @return legacy class name for FileFileTrailer#comparatorClassName
1902      */
1903     @Override
1904     public String getLegacyKeyComparatorName() {
1905       return "org.apache.hadoop.hbase.KeyValue$MetaKeyComparator";
1906     }
1907 
1908     @Override
1909     protected Object clone() throws CloneNotSupportedException {
1910       return new MetaComparator();
1911     }
1912 
1913     /**
1914      * Override the row key comparison to parse and compare the meta row key parts.
1915      */
1916     @Override
1917     protected int compareRowKey(final Cell l, final Cell r) {
1918       byte[] left = l.getRowArray();
1919       int loffset = l.getRowOffset();
1920       int llength = l.getRowLength();
1921       byte[] right = r.getRowArray();
1922       int roffset = r.getRowOffset();
1923       int rlength = r.getRowLength();
1924       return compareRows(left, loffset, llength, right, roffset, rlength);
1925     }
1926   }
1927 
1928   /**
1929    * Compare KeyValues.  When we compare KeyValues, we only compare the Key
1930    * portion.  This means two KeyValues with same Key but different Values are
1931    * considered the same as far as this Comparator is concerned.
1932    */
1933   public static class KVComparator implements RawComparator<Cell>, SamePrefixComparator<byte[]> {
1934 
1935     /**
1936      * The HFileV2 file format's trailer contains this class name.  We reinterpret this and
1937      * instantiate the appropriate comparator.
1938      * TODO: With V3 consider removing this.
1939      * @return legacy class name for FileFileTrailer#comparatorClassName
1940      */
1941     public String getLegacyKeyComparatorName() {
1942       return "org.apache.hadoop.hbase.KeyValue$KeyComparator";
1943     }
1944 
1945     @Override // RawComparator
1946     public int compare(byte[] l, int loff, int llen, byte[] r, int roff, int rlen) {
1947       return compareFlatKey(l,loff,llen, r,roff,rlen);
1948     }
1949 
1950     
1951     /**
1952      * Compares the only the user specified portion of a Key.  This is overridden by MetaComparator.
1953      * @param left
1954      * @param right
1955      * @return 0 if equal, <0 if left smaller, >0 if right smaller
1956      */
1957     protected int compareRowKey(final Cell left, final Cell right) {
1958       return Bytes.compareTo(
1959           left.getRowArray(),  left.getRowOffset(),  left.getRowLength(),
1960           right.getRowArray(), right.getRowOffset(), right.getRowLength());
1961     }
1962 
1963     /**
1964      * Compares left to right assuming that left,loffset,llength and right,roffset,rlength are
1965      * full KVs laid out in a flat byte[]s.
1966      * @param left
1967      * @param loffset
1968      * @param llength
1969      * @param right
1970      * @param roffset
1971      * @param rlength
1972      * @return  0 if equal, <0 if left smaller, >0 if right smaller
1973      */
1974     public int compareFlatKey(byte[] left, int loffset, int llength,
1975         byte[] right, int roffset, int rlength) {
1976       // Compare row
1977       short lrowlength = Bytes.toShort(left, loffset);
1978       short rrowlength = Bytes.toShort(right, roffset);
1979       int compare = compareRows(left, loffset + Bytes.SIZEOF_SHORT,
1980           lrowlength, right, roffset + Bytes.SIZEOF_SHORT, rrowlength);
1981       if (compare != 0) {
1982         return compare;
1983       }
1984 
1985       // Compare the rest of the two KVs without making any assumptions about
1986       // the common prefix. This function will not compare rows anyway, so we
1987       // don't need to tell it that the common prefix includes the row.
1988       return compareWithoutRow(0, left, loffset, llength, right, roffset,
1989           rlength, rrowlength);
1990     }
1991 
1992     public int compareFlatKey(byte[] left, byte[] right) {
1993       return compareFlatKey(left, 0, left.length, right, 0, right.length);
1994     }
1995 
1996     /**
1997      * Compares the Key of a cell -- with fields being more significant in this order:
1998      * rowkey, colfam/qual, timestamp, type, mvcc
1999      */
2000     public int compare(final Cell left, final Cell right) {
2001       // compare row
2002       int compare = compareRowKey(left, right);
2003       if (compare != 0) {
2004         return compare;
2005       }
2006 
2007       // compare vs minimum
2008       byte ltype = left.getTypeByte();
2009       byte rtype = right.getTypeByte();
2010       // If the column is not specified, the "minimum" key type appears the
2011       // latest in the sorted order, regardless of the timestamp. This is used
2012       // for specifying the last key/value in a given row, because there is no
2013       // "lexicographically last column" (it would be infinitely long). The
2014       // "maximum" key type does not need this behavior.
2015       int lcfqLen = left.getFamilyLength() + left.getQualifierLength() ;
2016       int rcfqLen = right.getFamilyLength() + right.getQualifierLength() ;
2017       if (lcfqLen == 0 && ltype == Type.Minimum.getCode()) {
2018         // left is "bigger", i.e. it appears later in the sorted order
2019         return 1;
2020       }
2021       if (rcfqLen == 0 && rtype == Type.Minimum.getCode()) {
2022         return -1;
2023       }
2024 
2025 
2026       // compare col family / col fam + qual
2027       // If left family size is not equal to right family size, we need not
2028       // compare the qualifiers.
2029       compare = Bytes.compareTo(
2030         left.getFamilyArray(),  left.getFamilyOffset(),  left.getFamilyLength(),
2031         right.getFamilyArray(), right.getFamilyOffset(), right.getFamilyLength());
2032       if (compare != 0) {
2033         return compare;
2034       }
2035 
2036       // Compare qualifier
2037       compare = Bytes.compareTo(
2038           left.getQualifierArray(), left.getQualifierOffset(), left.getQualifierLength(),
2039           right.getQualifierArray(), right.getQualifierOffset(), right.getQualifierLength());
2040       if (compare!= 0) {
2041         return compare;
2042       }
2043 
2044       // compare timestamp
2045       long ltimestamp = left.getTimestamp();
2046       long rtimestamp = right.getTimestamp();
2047       compare = compareTimestamps(ltimestamp, rtimestamp);
2048       if (compare != 0) {
2049         return compare;
2050       }
2051 
2052       // Compare types. Let the delete types sort ahead of puts; i.e. types
2053       // of higher numbers sort before those of lesser numbers. Maximum (255)
2054       // appears ahead of everything, and minimum (0) appears after
2055       // everything.
2056       compare = (0xff & rtype) - (0xff & ltype);
2057       if (compare != 0) {
2058         return compare;
2059       }
2060 
2061       // Negate following comparisons so later edits show up first
2062 
2063       // compare log replay tag value if there is any
2064       // when either keyvalue tagged with log replay sequence number, we need to compare them:
2065       // 1) when both keyvalues have the tag, then use the tag values for comparison
2066       // 2) when one has and the other doesn't have, the one without the log replay tag wins because
2067       // it means the edit isn't from recovery but new one coming from clients during recovery
2068       // 3) when both doesn't have, then skip to the next mvcc comparison
2069       long leftChangeSeqNum = getReplaySeqNum(left);
2070       long RightChangeSeqNum = getReplaySeqNum(right);
2071       if (leftChangeSeqNum != Long.MAX_VALUE || RightChangeSeqNum != Long.MAX_VALUE) {
2072         return Longs.compare(RightChangeSeqNum, leftChangeSeqNum);
2073       }
2074 
2075       // compare Mvcc Version
2076       return Longs.compare(right.getMvccVersion(), left.getMvccVersion());
2077     }
2078     
2079     /**
2080      * Return replay log sequence number for the cell
2081      * @param c
2082      * @return Long.MAX_VALUE if there is no LOG_REPLAY_TAG
2083      */
2084     private long getReplaySeqNum(final Cell c) {
2085       Tag tag = Tag.getTag(c.getTagsArray(), c.getTagsOffset(), c.getTagsLengthUnsigned(),
2086           TagType.LOG_REPLAY_TAG_TYPE);
2087 
2088       if(tag != null) {
2089         return Bytes.toLong(tag.getBuffer(), tag.getTagOffset(), tag.getTagLength());
2090       }
2091       return Long.MAX_VALUE;
2092     }
2093 
2094     public int compareTimestamps(final KeyValue left, final KeyValue right) {
2095       // Compare timestamps
2096       long ltimestamp = left.getTimestamp(left.getKeyLength());
2097       long rtimestamp = right.getTimestamp(right.getKeyLength());
2098       return compareTimestamps(ltimestamp, rtimestamp);
2099     }
2100 
2101     /**
2102      * @param left
2103      * @param right
2104      * @return Result comparing rows.
2105      */
2106     public int compareRows(final KeyValue left, final KeyValue right) {
2107       return compareRows(left.getBuffer(),left.getRowOffset(), left.getRowLength(),
2108       right.getBuffer(), right.getRowOffset(), right.getRowLength());
2109     }
2110 
2111     /**
2112      * Get the b[],o,l for left and right rowkey portions and compare.
2113      * @param left
2114      * @param loffset
2115      * @param llength
2116      * @param right
2117      * @param roffset
2118      * @param rlength
2119      * @return 0 if equal, <0 if left smaller, >0 if right smaller
2120      */
2121     public int compareRows(byte [] left, int loffset, int llength,
2122         byte [] right, int roffset, int rlength) {
2123       return Bytes.compareTo(left, loffset, llength, right, roffset, rlength);
2124     }
2125 
2126     int compareColumns(final KeyValue left, final short lrowlength,
2127         final KeyValue right, final short rrowlength) {
2128       int lfoffset = left.getFamilyOffset(lrowlength);
2129       int rfoffset = right.getFamilyOffset(rrowlength);
2130       int lclength = left.getTotalColumnLength(lrowlength,lfoffset);
2131       int rclength = right.getTotalColumnLength(rrowlength, rfoffset);
2132       int lfamilylength = left.getFamilyLength(lfoffset);
2133       int rfamilylength = right.getFamilyLength(rfoffset);
2134       return compareColumns(left.getBuffer(), lfoffset,
2135           lclength, lfamilylength,
2136         right.getBuffer(), rfoffset, rclength, rfamilylength);
2137     }
2138 
2139     protected int compareColumns(
2140         byte [] left, int loffset, int llength, final int lfamilylength,
2141         byte [] right, int roffset, int rlength, final int rfamilylength) {
2142       // Compare family portion first.
2143       int diff = Bytes.compareTo(left, loffset, lfamilylength,
2144         right, roffset, rfamilylength);
2145       if (diff != 0) {
2146         return diff;
2147       }
2148       // Compare qualifier portion
2149       return Bytes.compareTo(left, loffset + lfamilylength,
2150         llength - lfamilylength,
2151         right, roffset + rfamilylength, rlength - rfamilylength);
2152       }
2153 
2154     static int compareTimestamps(final long ltimestamp, final long rtimestamp) {
2155       // The below older timestamps sorting ahead of newer timestamps looks
2156       // wrong but it is intentional. This way, newer timestamps are first
2157       // found when we iterate over a memstore and newer versions are the
2158       // first we trip over when reading from a store file.
2159       if (ltimestamp < rtimestamp) {
2160         return 1;
2161       } else if (ltimestamp > rtimestamp) {
2162         return -1;
2163       }
2164       return 0;
2165     }
2166 
2167     /**
2168      * Overridden
2169      * @param commonPrefix
2170      * @param left
2171      * @param loffset
2172      * @param llength
2173      * @param right
2174      * @param roffset
2175      * @param rlength
2176      * @return 0 if equal, <0 if left smaller, >0 if right smaller
2177      */
2178     @Override // SamePrefixComparator
2179     public int compareIgnoringPrefix(int commonPrefix, byte[] left,
2180         int loffset, int llength, byte[] right, int roffset, int rlength) {
2181       // Compare row
2182       short lrowlength = Bytes.toShort(left, loffset);
2183       short rrowlength;
2184 
2185       int comparisonResult = 0;
2186       if (commonPrefix < ROW_LENGTH_SIZE) {
2187         // almost nothing in common
2188         rrowlength = Bytes.toShort(right, roffset);
2189         comparisonResult = compareRows(left, loffset + ROW_LENGTH_SIZE,
2190             lrowlength, right, roffset + ROW_LENGTH_SIZE, rrowlength);
2191       } else { // the row length is the same
2192         rrowlength = lrowlength;
2193         if (commonPrefix < ROW_LENGTH_SIZE + rrowlength) {
2194           // The rows are not the same. Exclude the common prefix and compare
2195           // the rest of the two rows.
2196           int common = commonPrefix - ROW_LENGTH_SIZE;
2197           comparisonResult = compareRows(
2198               left, loffset + common + ROW_LENGTH_SIZE, lrowlength - common,
2199               right, roffset + common + ROW_LENGTH_SIZE, rrowlength - common);
2200         }
2201       }
2202       if (comparisonResult != 0) {
2203         return comparisonResult;
2204       }
2205 
2206       assert lrowlength == rrowlength;
2207       return compareWithoutRow(commonPrefix, left, loffset, llength, right,
2208           roffset, rlength, lrowlength);
2209     }
2210 
2211     /**
2212      * Compare columnFamily, qualifier, timestamp, and key type (everything
2213      * except the row). This method is used both in the normal comparator and
2214      * the "same-prefix" comparator. Note that we are assuming that row portions
2215      * of both KVs have already been parsed and found identical, and we don't
2216      * validate that assumption here.
2217      * @param commonPrefix
2218      *          the length of the common prefix of the two key-values being
2219      *          compared, including row length and row
2220      */
2221     private int compareWithoutRow(int commonPrefix, byte[] left, int loffset,
2222         int llength, byte[] right, int roffset, int rlength, short rowlength) {
2223       /***
2224        * KeyValue Format and commonLength:
2225        * |_keyLen_|_valLen_|_rowLen_|_rowKey_|_famiLen_|_fami_|_Quali_|....
2226        * ------------------|-------commonLength--------|--------------
2227        */
2228       int commonLength = ROW_LENGTH_SIZE + FAMILY_LENGTH_SIZE + rowlength;
2229 
2230       // commonLength + TIMESTAMP_TYPE_SIZE
2231       int commonLengthWithTSAndType = TIMESTAMP_TYPE_SIZE + commonLength;
2232       // ColumnFamily + Qualifier length.
2233       int lcolumnlength = llength - commonLengthWithTSAndType;
2234       int rcolumnlength = rlength - commonLengthWithTSAndType;
2235 
2236       byte ltype = left[loffset + (llength - 1)];
2237       byte rtype = right[roffset + (rlength - 1)];
2238 
2239       // If the column is not specified, the "minimum" key type appears the
2240       // latest in the sorted order, regardless of the timestamp. This is used
2241       // for specifying the last key/value in a given row, because there is no
2242       // "lexicographically last column" (it would be infinitely long). The
2243       // "maximum" key type does not need this behavior.
2244       if (lcolumnlength == 0 && ltype == Type.Minimum.getCode()) {
2245         // left is "bigger", i.e. it appears later in the sorted order
2246         return 1;
2247       }
2248       if (rcolumnlength == 0 && rtype == Type.Minimum.getCode()) {
2249         return -1;
2250       }
2251 
2252       int lfamilyoffset = commonLength + loffset;
2253       int rfamilyoffset = commonLength + roffset;
2254 
2255       // Column family length.
2256       int lfamilylength = left[lfamilyoffset - 1];
2257       int rfamilylength = right[rfamilyoffset - 1];
2258       // If left family size is not equal to right family size, we need not
2259       // compare the qualifiers.
2260       boolean sameFamilySize = (lfamilylength == rfamilylength);
2261       int common = 0;
2262       if (commonPrefix > 0) {
2263         common = Math.max(0, commonPrefix - commonLength);
2264         if (!sameFamilySize) {
2265           // Common should not be larger than Math.min(lfamilylength,
2266           // rfamilylength).
2267           common = Math.min(common, Math.min(lfamilylength, rfamilylength));
2268         } else {
2269           common = Math.min(common, Math.min(lcolumnlength, rcolumnlength));
2270         }
2271       }
2272       if (!sameFamilySize) {
2273         // comparing column family is enough.
2274         return Bytes.compareTo(left, lfamilyoffset + common, lfamilylength
2275             - common, right, rfamilyoffset + common, rfamilylength - common);
2276       }
2277       // Compare family & qualifier together.
2278       final int comparison = Bytes.compareTo(left, lfamilyoffset + common,
2279           lcolumnlength - common, right, rfamilyoffset + common,
2280           rcolumnlength - common);
2281       if (comparison != 0) {
2282         return comparison;
2283       }
2284 
2285       ////
2286       // Next compare timestamps.
2287       long ltimestamp = Bytes.toLong(left,
2288           loffset + (llength - TIMESTAMP_TYPE_SIZE));
2289       long rtimestamp = Bytes.toLong(right,
2290           roffset + (rlength - TIMESTAMP_TYPE_SIZE));
2291       int compare = compareTimestamps(ltimestamp, rtimestamp);
2292       if (compare != 0) {
2293         return compare;
2294       }
2295 
2296       // Compare types. Let the delete types sort ahead of puts; i.e. types
2297       // of higher numbers sort before those of lesser numbers. Maximum (255)
2298       // appears ahead of everything, and minimum (0) appears after
2299       // everything.
2300       return (0xff & rtype) - (0xff & ltype);
2301     }
2302 
2303     /**
2304      * Compares the row and column of two keyvalues for equality
2305      * @param left
2306      * @param right
2307      * @return True if same row and column.
2308      */
2309     public boolean matchingRowColumn(final KeyValue left,
2310         final KeyValue right) {
2311       short lrowlength = left.getRowLength();
2312       short rrowlength = right.getRowLength();
2313 
2314       // TsOffset = end of column data. just comparing Row+CF length of each
2315       if ((left.getTimestampOffset() - left.getOffset()) !=
2316           (right.getTimestampOffset() - right.getOffset())) {
2317         return false;
2318       }
2319 
2320       if (!matchingRows(left, lrowlength, right, rrowlength)) {
2321         return false;
2322       }
2323 
2324       int lfoffset = left.getFamilyOffset(lrowlength);
2325       int rfoffset = right.getFamilyOffset(rrowlength);
2326       int lclength = left.getTotalColumnLength(lrowlength,lfoffset);
2327       int rclength = right.getTotalColumnLength(rrowlength, rfoffset);
2328       int lfamilylength = left.getFamilyLength(lfoffset);
2329       int rfamilylength = right.getFamilyLength(rfoffset);
2330       int ccRes = compareColumns(left.getBuffer(), lfoffset, lclength, lfamilylength,
2331           right.getBuffer(), rfoffset, rclength, rfamilylength);
2332       return ccRes == 0;
2333     }
2334 
2335     /**
2336      * Compares the row of two keyvalues for equality
2337      * @param left
2338      * @param right
2339      * @return True if rows match.
2340      */
2341     public boolean matchingRows(final KeyValue left, final KeyValue right) {
2342       short lrowlength = left.getRowLength();
2343       short rrowlength = right.getRowLength();
2344       return matchingRows(left, lrowlength, right, rrowlength);
2345     }
2346 
2347     /**
2348      * @param left
2349      * @param lrowlength
2350      * @param right
2351      * @param rrowlength
2352      * @return True if rows match.
2353      */
2354     private boolean matchingRows(final KeyValue left, final short lrowlength,
2355         final KeyValue right, final short rrowlength) {
2356       return lrowlength == rrowlength &&
2357           matchingRows(left.getBuffer(), left.getRowOffset(), lrowlength,
2358               right.getBuffer(), right.getRowOffset(), rrowlength);
2359     }
2360 
2361     /**
2362      * Compare rows. Just calls Bytes.equals, but it's good to have this encapsulated.
2363      * @param left Left row array.
2364      * @param loffset Left row offset.
2365      * @param llength Left row length.
2366      * @param right Right row array.
2367      * @param roffset Right row offset.
2368      * @param rlength Right row length.
2369      * @return Whether rows are the same row.
2370      */
2371     public boolean matchingRows(final byte [] left, final int loffset, final int llength,
2372         final byte [] right, final int roffset, final int rlength) {
2373       return Bytes.equals(left, loffset, llength, right, roffset, rlength);
2374     }
2375 
2376     public byte[] calcIndexKey(byte[] lastKeyOfPreviousBlock, byte[] firstKeyInBlock) {
2377       byte[] fakeKey = getShortMidpointKey(lastKeyOfPreviousBlock, firstKeyInBlock);
2378       if (compareFlatKey(fakeKey, firstKeyInBlock) > 0) {
2379         LOG.error("Unexpected getShortMidpointKey result, fakeKey:"
2380             + Bytes.toStringBinary(fakeKey) + ", firstKeyInBlock:"
2381             + Bytes.toStringBinary(firstKeyInBlock));
2382         return firstKeyInBlock;
2383       }
2384       if (lastKeyOfPreviousBlock != null && compareFlatKey(lastKeyOfPreviousBlock, fakeKey) >= 0) {
2385         LOG.error("Unexpected getShortMidpointKey result, lastKeyOfPreviousBlock:" +
2386             Bytes.toStringBinary(lastKeyOfPreviousBlock) + ", fakeKey:" +
2387             Bytes.toStringBinary(fakeKey));
2388         return firstKeyInBlock;
2389       }
2390       return fakeKey;
2391     }
2392 
2393     /**
2394      * This is a HFile block index key optimization.
2395      * @param leftKey
2396      * @param rightKey
2397      * @return 0 if equal, <0 if left smaller, >0 if right smaller
2398      */
2399     public byte[] getShortMidpointKey(final byte[] leftKey, final byte[] rightKey) {
2400       if (rightKey == null) {
2401         throw new IllegalArgumentException("rightKey can not be null");
2402       }
2403       if (leftKey == null) {
2404         return Arrays.copyOf(rightKey, rightKey.length);
2405       }
2406       if (compareFlatKey(leftKey, rightKey) >= 0) {
2407         throw new IllegalArgumentException("Unexpected input, leftKey:" + Bytes.toString(leftKey)
2408           + ", rightKey:" + Bytes.toString(rightKey));
2409       }
2410 
2411       short leftRowLength = Bytes.toShort(leftKey, 0);
2412       short rightRowLength = Bytes.toShort(rightKey, 0);
2413       int leftCommonLength = ROW_LENGTH_SIZE + FAMILY_LENGTH_SIZE + leftRowLength;
2414       int rightCommonLength = ROW_LENGTH_SIZE + FAMILY_LENGTH_SIZE + rightRowLength;
2415       int leftCommonLengthWithTSAndType = TIMESTAMP_TYPE_SIZE + leftCommonLength;
2416       int rightCommonLengthWithTSAndType = TIMESTAMP_TYPE_SIZE + rightCommonLength;
2417       int leftColumnLength = leftKey.length - leftCommonLengthWithTSAndType;
2418       int rightColumnLength = rightKey.length - rightCommonLengthWithTSAndType;
2419       // rows are equal
2420       if (leftRowLength == rightRowLength && compareRows(leftKey, ROW_LENGTH_SIZE, leftRowLength,
2421         rightKey, ROW_LENGTH_SIZE, rightRowLength) == 0) {
2422         // Compare family & qualifier together.
2423         int comparison = Bytes.compareTo(leftKey, leftCommonLength, leftColumnLength, rightKey,
2424           rightCommonLength, rightColumnLength);
2425         // same with "row + family + qualifier", return rightKey directly
2426         if (comparison == 0) {
2427           return Arrays.copyOf(rightKey, rightKey.length);
2428         }
2429         // "family + qualifier" are different, generate a faked key per rightKey
2430         byte[] newKey = Arrays.copyOf(rightKey, rightKey.length);
2431         Bytes.putLong(newKey, rightKey.length - TIMESTAMP_TYPE_SIZE, HConstants.LATEST_TIMESTAMP);
2432         Bytes.putByte(newKey, rightKey.length - TYPE_SIZE, Type.Maximum.getCode());
2433         return newKey;
2434       }
2435       // rows are different
2436       short minLength = leftRowLength < rightRowLength ? leftRowLength : rightRowLength;
2437       short diffIdx = 0;
2438       while (diffIdx < minLength
2439           && leftKey[ROW_LENGTH_SIZE + diffIdx] == rightKey[ROW_LENGTH_SIZE + diffIdx]) {
2440         diffIdx++;
2441       }
2442       byte[] newRowKey = null;
2443       if (diffIdx >= minLength) {
2444         // leftKey's row is prefix of rightKey's.
2445         newRowKey = new byte[diffIdx + 1];
2446         System.arraycopy(rightKey, ROW_LENGTH_SIZE, newRowKey, 0, diffIdx + 1);
2447       } else {
2448         int diffByte = leftKey[ROW_LENGTH_SIZE + diffIdx];
2449         if ((0xff & diffByte) < 0xff && (diffByte + 1) <
2450             (rightKey[ROW_LENGTH_SIZE + diffIdx] & 0xff)) {
2451           newRowKey = new byte[diffIdx + 1];
2452           System.arraycopy(leftKey, ROW_LENGTH_SIZE, newRowKey, 0, diffIdx);
2453           newRowKey[diffIdx] = (byte) (diffByte + 1);
2454         } else {
2455           newRowKey = new byte[diffIdx + 1];
2456           System.arraycopy(rightKey, ROW_LENGTH_SIZE, newRowKey, 0, diffIdx + 1);
2457         }
2458       }
2459       return new KeyValue(newRowKey, null, null, HConstants.LATEST_TIMESTAMP,
2460         Type.Maximum).getKey();
2461     }
2462 
2463     @Override
2464     protected Object clone() throws CloneNotSupportedException {
2465       return new KVComparator();
2466     }
2467 
2468   }
2469 
2470   /**
2471    * Creates a KeyValue that is last on the specified row id. That is,
2472    * every other possible KeyValue for the given row would compareTo()
2473    * less than the result of this call.
2474    * @param row row key
2475    * @return Last possible KeyValue on passed <code>row</code>
2476    */
2477   public static KeyValue createLastOnRow(final byte[] row) {
2478     return new KeyValue(row, null, null, HConstants.LATEST_TIMESTAMP, Type.Minimum);
2479   }
2480 
2481   /**
2482    * Create a KeyValue that is smaller than all other possible KeyValues
2483    * for the given row. That is any (valid) KeyValue on 'row' would sort
2484    * _after_ the result.
2485    *
2486    * @param row - row key (arbitrary byte array)
2487    * @return First possible KeyValue on passed <code>row</code>
2488    */
2489   public static KeyValue createFirstOnRow(final byte [] row) {
2490     return createFirstOnRow(row, HConstants.LATEST_TIMESTAMP);
2491   }
2492 
2493   /**
2494    * Create a KeyValue that is smaller than all other possible KeyValues
2495    * for the given row. That is any (valid) KeyValue on 'row' would sort
2496    * _after_ the result.
2497    *
2498    * @param row - row key (arbitrary byte array)
2499    * @return First possible KeyValue on passed <code>row</code>
2500    */
2501   public static KeyValue createFirstOnRow(final byte [] row, int roffset, short rlength) {
2502     return new KeyValue(row, roffset, rlength,
2503         null, 0, 0, null, 0, 0, HConstants.LATEST_TIMESTAMP, Type.Maximum, null, 0, 0);
2504   }
2505 
2506   /**
2507    * Creates a KeyValue that is smaller than all other KeyValues that
2508    * are older than the passed timestamp.
2509    * @param row - row key (arbitrary byte array)
2510    * @param ts - timestamp
2511    * @return First possible key on passed <code>row</code> and timestamp.
2512    */
2513   public static KeyValue createFirstOnRow(final byte [] row,
2514       final long ts) {
2515     return new KeyValue(row, null, null, ts, Type.Maximum);
2516   }
2517 
2518   /**
2519    * Create a KeyValue for the specified row, family and qualifier that would be
2520    * smaller than all other possible KeyValues that have the same row,family,qualifier.
2521    * Used for seeking.
2522    * @param row - row key (arbitrary byte array)
2523    * @param family - family name
2524    * @param qualifier - column qualifier
2525    * @return First possible key on passed <code>row</code>, and column.
2526    */
2527   public static KeyValue createFirstOnRow(final byte [] row, final byte [] family,
2528       final byte [] qualifier) {
2529     return new KeyValue(row, family, qualifier, HConstants.LATEST_TIMESTAMP, Type.Maximum);
2530   }
2531 
2532   /**
2533    * Create a Delete Family KeyValue for the specified row and family that would
2534    * be smaller than all other possible Delete Family KeyValues that have the
2535    * same row and family.
2536    * Used for seeking.
2537    * @param row - row key (arbitrary byte array)
2538    * @param family - family name
2539    * @return First Delete Family possible key on passed <code>row</code>.
2540    */
2541   public static KeyValue createFirstDeleteFamilyOnRow(final byte [] row,
2542       final byte [] family) {
2543     return new KeyValue(row, family, null, HConstants.LATEST_TIMESTAMP,
2544         Type.DeleteFamily);
2545   }
2546 
2547   /**
2548    * @param row - row key (arbitrary byte array)
2549    * @param f - family name
2550    * @param q - column qualifier
2551    * @param ts - timestamp
2552    * @return First possible key on passed <code>row</code>, column and timestamp
2553    */
2554   public static KeyValue createFirstOnRow(final byte [] row, final byte [] f,
2555       final byte [] q, final long ts) {
2556     return new KeyValue(row, f, q, ts, Type.Maximum);
2557   }
2558 
2559   /**
2560    * Create a KeyValue for the specified row, family and qualifier that would be
2561    * smaller than all other possible KeyValues that have the same row,
2562    * family, qualifier.
2563    * Used for seeking.
2564    * @param row row key
2565    * @param roffset row offset
2566    * @param rlength row length
2567    * @param family family name
2568    * @param foffset family offset
2569    * @param flength family length
2570    * @param qualifier column qualifier
2571    * @param qoffset qualifier offset
2572    * @param qlength qualifier length
2573    * @return First possible key on passed Row, Family, Qualifier.
2574    */
2575   public static KeyValue createFirstOnRow(final byte [] row,
2576       final int roffset, final int rlength, final byte [] family,
2577       final int foffset, final int flength, final byte [] qualifier,
2578       final int qoffset, final int qlength) {
2579     return new KeyValue(row, roffset, rlength, family,
2580         foffset, flength, qualifier, qoffset, qlength,
2581         HConstants.LATEST_TIMESTAMP, Type.Maximum, null, 0, 0);
2582   }
2583 
2584   /**
2585    * Create a KeyValue for the specified row, family and qualifier that would be
2586    * smaller than all other possible KeyValues that have the same row,
2587    * family, qualifier.
2588    * Used for seeking.
2589    *
2590    * @param buffer the buffer to use for the new <code>KeyValue</code> object
2591    * @param row the value key
2592    * @param family family name
2593    * @param qualifier column qualifier
2594    *
2595    * @return First possible key on passed Row, Family, Qualifier.
2596    *
2597    * @throws IllegalArgumentException The resulting <code>KeyValue</code> object would be larger
2598    * than the provided buffer or than <code>Integer.MAX_VALUE</code>
2599    */
2600   public static KeyValue createFirstOnRow(byte [] buffer, final byte [] row,
2601       final byte [] family, final byte [] qualifier)
2602           throws IllegalArgumentException {
2603 
2604     return createFirstOnRow(buffer, 0, row, 0, row.length,
2605         family, 0, family.length,
2606         qualifier, 0, qualifier.length);
2607   }
2608 
2609   /**
2610    * Create a KeyValue for the specified row, family and qualifier that would be
2611    * smaller than all other possible KeyValues that have the same row,
2612    * family, qualifier.
2613    * Used for seeking.
2614    *
2615    * @param buffer the buffer to use for the new <code>KeyValue</code> object
2616    * @param boffset buffer offset
2617    * @param row the value key
2618    * @param roffset row offset
2619    * @param rlength row length
2620    * @param family family name
2621    * @param foffset family offset
2622    * @param flength family length
2623    * @param qualifier column qualifier
2624    * @param qoffset qualifier offset
2625    * @param qlength qualifier length
2626    *
2627    * @return First possible key on passed Row, Family, Qualifier.
2628    *
2629    * @throws IllegalArgumentException The resulting <code>KeyValue</code> object would be larger
2630    * than the provided buffer or than <code>Integer.MAX_VALUE</code>
2631    */
2632   public static KeyValue createFirstOnRow(byte [] buffer, final int boffset,
2633       final byte [] row, final int roffset, final int rlength,
2634       final byte [] family, final int foffset, final int flength,
2635       final byte [] qualifier, final int qoffset, final int qlength)
2636           throws IllegalArgumentException {
2637 
2638     long lLength = getKeyValueDataStructureSize(rlength, flength, qlength, 0);
2639 
2640     if (lLength > Integer.MAX_VALUE) {
2641       throw new IllegalArgumentException("KeyValue length " + lLength + " > " + Integer.MAX_VALUE);
2642     }
2643     int iLength = (int) lLength;
2644     if (buffer.length - boffset < iLength) {
2645       throw new IllegalArgumentException("Buffer size " + (buffer.length - boffset) + " < " +
2646           iLength);
2647     }
2648 
2649     int len = writeByteArray(buffer, boffset, row, roffset, rlength, family, foffset, flength,
2650         qualifier, qoffset, qlength, HConstants.LATEST_TIMESTAMP, KeyValue.Type.Maximum,
2651         null, 0, 0, null);
2652     return new KeyValue(buffer, boffset, len);
2653   }
2654 
2655   /**
2656    * Create a KeyValue for the specified row, family and qualifier that would be
2657    * larger than or equal to all other possible KeyValues that have the same
2658    * row, family, qualifier.
2659    * Used for reseeking.
2660    * @param row row key
2661    * @param roffset row offset
2662    * @param rlength row length
2663    * @param family family name
2664    * @param foffset family offset
2665    * @param flength family length
2666    * @param qualifier column qualifier
2667    * @param qoffset qualifier offset
2668    * @param qlength qualifier length
2669    * @return Last possible key on passed row, family, qualifier.
2670    */
2671   public static KeyValue createLastOnRow(final byte [] row,
2672       final int roffset, final int rlength, final byte [] family,
2673       final int foffset, final int flength, final byte [] qualifier,
2674       final int qoffset, final int qlength) {
2675     return new KeyValue(row, roffset, rlength, family,
2676         foffset, flength, qualifier, qoffset, qlength,
2677         HConstants.OLDEST_TIMESTAMP, Type.Minimum, null, 0, 0);
2678   }
2679 
2680   /**
2681    * Similar to {@link #createLastOnRow(byte[], int, int, byte[], int, int,
2682    * byte[], int, int)} but creates the last key on the row/column of this KV
2683    * (the value part of the returned KV is always empty). Used in creating
2684    * "fake keys" for the multi-column Bloom filter optimization to skip the
2685    * row/column we already know is not in the file.
2686    * @return the last key on the row/column of the given key-value pair
2687    */
2688   public KeyValue createLastOnRowCol() {
2689     return new KeyValue(
2690         bytes, getRowOffset(), getRowLength(),
2691         bytes, getFamilyOffset(), getFamilyLength(),
2692         bytes, getQualifierOffset(), getQualifierLength(),
2693         HConstants.OLDEST_TIMESTAMP, Type.Minimum, null, 0, 0);
2694   }
2695 
2696   /**
2697    * Creates the first KV with the row/family/qualifier of this KV and the
2698    * given timestamp. Uses the "maximum" KV type that guarantees that the new
2699    * KV is the lowest possible for this combination of row, family, qualifier,
2700    * and timestamp. This KV's own timestamp is ignored. While this function
2701    * copies the value from this KV, it is normally used on key-only KVs.
2702    */
2703   public KeyValue createFirstOnRowColTS(long ts) {
2704     return new KeyValue(
2705         bytes, getRowOffset(), getRowLength(),
2706         bytes, getFamilyOffset(), getFamilyLength(),
2707         bytes, getQualifierOffset(), getQualifierLength(),
2708         ts, Type.Maximum, bytes, getValueOffset(), getValueLength());
2709   }
2710 
2711   /**
2712    * @param b
2713    * @return A KeyValue made of a byte array that holds the key-only part.
2714    * Needed to convert hfile index members to KeyValues.
2715    */
2716   public static KeyValue createKeyValueFromKey(final byte [] b) {
2717     return createKeyValueFromKey(b, 0, b.length);
2718   }
2719 
2720   /**
2721    * @param bb
2722    * @return A KeyValue made of a byte buffer that holds the key-only part.
2723    * Needed to convert hfile index members to KeyValues.
2724    */
2725   public static KeyValue createKeyValueFromKey(final ByteBuffer bb) {
2726     return createKeyValueFromKey(bb.array(), bb.arrayOffset(), bb.limit());
2727   }
2728 
2729   /**
2730    * @param b
2731    * @param o
2732    * @param l
2733    * @return A KeyValue made of a byte array that holds the key-only part.
2734    * Needed to convert hfile index members to KeyValues.
2735    */
2736   public static KeyValue createKeyValueFromKey(final byte [] b, final int o,
2737       final int l) {
2738     byte [] newb = new byte[l + ROW_OFFSET];
2739     System.arraycopy(b, o, newb, ROW_OFFSET, l);
2740     Bytes.putInt(newb, 0, l);
2741     Bytes.putInt(newb, Bytes.SIZEOF_INT, 0);
2742     return new KeyValue(newb);
2743   }
2744 
2745   /**
2746    * @param in Where to read bytes from.  Creates a byte array to hold the KeyValue
2747    * backing bytes copied from the steam.
2748    * @return KeyValue created by deserializing from <code>in</code> OR if we find a length
2749    * of zero, we will return null which can be useful marking a stream as done.
2750    * @throws IOException
2751    */
2752   public static KeyValue create(final DataInput in) throws IOException {
2753     return create(in.readInt(), in);
2754   }
2755 
2756   /**
2757    * Create a KeyValue reading <code>length</code> from <code>in</code>
2758    * @param length
2759    * @param in
2760    * @return Created KeyValue OR if we find a length of zero, we will return null which
2761    * can be useful marking a stream as done.
2762    * @throws IOException
2763    */
2764   public static KeyValue create(int length, final DataInput in) throws IOException {
2765 
2766     if (length <= 0) {
2767       if (length == 0) return null;
2768       throw new IOException("Failed read " + length + " bytes, stream corrupt?");
2769     }
2770 
2771     // This is how the old Writables.readFrom used to deserialize.  Didn't even vint.
2772     byte [] bytes = new byte[length];
2773     in.readFully(bytes);
2774     return new KeyValue(bytes, 0, length);
2775   }
2776   
2777   /**
2778    * Create a new KeyValue by copying existing cell and adding new tags
2779    * @param c
2780    * @param newTags
2781    * @return a new KeyValue instance with new tags
2782    */
2783   public static KeyValue cloneAndAddTags(Cell c, List<Tag> newTags) {
2784     List<Tag> existingTags = null;
2785     if(c.getTagsLengthUnsigned() > 0) {
2786       existingTags = Tag.asList(c.getTagsArray(), c.getTagsOffset(), c.getTagsLengthUnsigned());
2787       existingTags.addAll(newTags);
2788     } else {
2789       existingTags = newTags;
2790     }
2791     return new KeyValue(c.getRowArray(), c.getRowOffset(), (int)c.getRowLength(),
2792       c.getFamilyArray(), c.getFamilyOffset(), (int)c.getFamilyLength(), 
2793       c.getQualifierArray(), c.getQualifierOffset(), (int) c.getQualifierLength(), 
2794       c.getTimestamp(), Type.codeToType(c.getTypeByte()), c.getValueArray(), c.getValueOffset(), 
2795       c.getValueLength(), existingTags);
2796   }
2797 
2798   /**
2799    * Create a KeyValue reading from the raw InputStream.
2800    * Named <code>iscreate</code> so doesn't clash with {@link #create(DataInput)}
2801    * @param in
2802    * @return Created KeyValue OR if we find a length of zero, we will return null which
2803    * can be useful marking a stream as done.
2804    * @throws IOException
2805    */
2806   public static KeyValue iscreate(final InputStream in) throws IOException {
2807     byte [] intBytes = new byte[Bytes.SIZEOF_INT];
2808     int bytesRead = 0;
2809     while (bytesRead < intBytes.length) {
2810       int n = in.read(intBytes, bytesRead, intBytes.length - bytesRead);
2811       if (n < 0) {
2812         if (bytesRead == 0) return null; // EOF at start is ok
2813         throw new IOException("Failed read of int, read " + bytesRead + " bytes");
2814       }
2815       bytesRead += n;
2816     }
2817     // TODO: perhaps some sanity check is needed here.
2818     byte [] bytes = new byte[Bytes.toInt(intBytes)];
2819     IOUtils.readFully(in, bytes, 0, bytes.length);
2820     return new KeyValue(bytes, 0, bytes.length);
2821   }
2822 
2823   /**
2824    * Write out a KeyValue in the manner in which we used to when KeyValue was a Writable.
2825    * @param kv
2826    * @param out
2827    * @return Length written on stream
2828    * @throws IOException
2829    * @see #create(DataInput) for the inverse function
2830    */
2831   public static long write(final KeyValue kv, final DataOutput out) throws IOException {
2832     // This is how the old Writables write used to serialize KVs.  Need to figure way to make it
2833     // work for all implementations.
2834     int length = kv.getLength();
2835     out.writeInt(length);
2836     out.write(kv.getBuffer(), kv.getOffset(), length);
2837     return length + Bytes.SIZEOF_INT;
2838   }
2839 
2840   /**
2841    * Write out a KeyValue in the manner in which we used to when KeyValue was a Writable but do
2842    * not require a {@link DataOutput}, just take plain {@link OutputStream}
2843    * Named <code>oswrite</code> so does not clash with {@link #write(KeyValue, DataOutput)}
2844    * @param kv
2845    * @param out
2846    * @return Length written on stream
2847    * @throws IOException
2848    * @see #create(DataInput) for the inverse function
2849    * @see #write(KeyValue, DataOutput)
2850    */
2851   @Deprecated
2852   public static long oswrite(final KeyValue kv, final OutputStream out)
2853       throws IOException {
2854     int length = kv.getLength();
2855     // This does same as DataOuput#writeInt (big-endian, etc.)
2856     out.write(Bytes.toBytes(length));
2857     out.write(kv.getBuffer(), kv.getOffset(), length);
2858     return length + Bytes.SIZEOF_INT;
2859   }
2860 
2861   /**
2862    * Write out a KeyValue in the manner in which we used to when KeyValue was a Writable but do
2863    * not require a {@link DataOutput}, just take plain {@link OutputStream}
2864    * Named <code>oswrite</code> so does not clash with {@link #write(KeyValue, DataOutput)}
2865    * @param kv
2866    * @param out
2867    * @param withTags
2868    * @return Length written on stream
2869    * @throws IOException
2870    * @see #create(DataInput) for the inverse function
2871    * @see #write(KeyValue, DataOutput)
2872    */
2873   public static long oswrite(final KeyValue kv, final OutputStream out, final boolean withTags)
2874       throws IOException {
2875     int length = kv.getLength();
2876     if (!withTags) {
2877       length = kv.getKeyLength() + kv.getValueLength() + KEYVALUE_INFRASTRUCTURE_SIZE;
2878     }
2879     // This does same as DataOuput#writeInt (big-endian, etc.)
2880     out.write(Bytes.toBytes(length));
2881     out.write(kv.getBuffer(), kv.getOffset(), length);
2882     return length + Bytes.SIZEOF_INT;
2883   }
2884 
2885   /**
2886    * Comparator that compares row component only of a KeyValue.
2887    */
2888   public static class RowOnlyComparator implements Comparator<KeyValue> {
2889     final KVComparator comparator;
2890 
2891     public RowOnlyComparator(final KVComparator c) {
2892       this.comparator = c;
2893     }
2894 
2895     public int compare(KeyValue left, KeyValue right) {
2896       return comparator.compareRows(left, right);
2897     }
2898   }
2899 
2900 
2901   /**
2902    * Avoids redundant comparisons for better performance.
2903    * 
2904    * TODO get rid of this wart
2905    */
2906   public interface SamePrefixComparator<T> {
2907     /**
2908      * Compare two keys assuming that the first n bytes are the same.
2909      * @param commonPrefix How many bytes are the same.
2910      */
2911     int compareIgnoringPrefix(
2912       int commonPrefix, byte[] left, int loffset, int llength, byte[] right, int roffset, int rlength
2913     );
2914   }
2915 
2916   /**
2917    * This is a TEST only Comparator used in TestSeekTo and TestReseekTo.
2918    */
2919   public static class RawBytesComparator extends KVComparator {
2920     /**
2921      * The HFileV2 file format's trailer contains this class name.  We reinterpret this and
2922      * instantiate the appropriate comparator.
2923      * TODO: With V3 consider removing this.
2924      * @return legacy class name for FileFileTrailer#comparatorClassName
2925      */
2926     public String getLegacyKeyComparatorName() {
2927       return "org.apache.hadoop.hbase.util.Bytes$ByteArrayComparator";
2928     }
2929 
2930     public int compareFlatKey(byte[] left, int loffset, int llength, byte[] right,
2931         int roffset, int rlength) {
2932       return Bytes.BYTES_RAWCOMPARATOR.compare(left,  loffset, llength, right, roffset, rlength);
2933     }
2934 
2935     public byte[] calcIndexKey(byte[] lastKeyOfPreviousBlock, byte[] firstKeyInBlock) {
2936       return firstKeyInBlock;
2937     }
2938 
2939   }
2940 
2941   /**
2942    * HeapSize implementation
2943    *
2944    * We do not count the bytes in the rowCache because it should be empty for a KeyValue in the
2945    * MemStore.
2946    */
2947   @Override
2948   public long heapSize() {
2949     int sum = 0;
2950     sum += ClassSize.OBJECT;// the KeyValue object itself
2951     sum += ClassSize.REFERENCE;// pointer to "bytes"
2952     sum += ClassSize.align(ClassSize.ARRAY);// "bytes"
2953     sum += ClassSize.align(length);// number of bytes of data in the "bytes" array
2954     sum += 2 * Bytes.SIZEOF_INT;// offset, length
2955     sum += Bytes.SIZEOF_LONG;// memstoreTS
2956     return ClassSize.align(sum);
2957   }
2958 }