View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.client;
20  
21  import java.nio.ByteBuffer;
22  import java.util.ArrayList;
23  import java.util.Arrays;
24  import java.util.HashMap;
25  import java.util.List;
26  import java.util.Map;
27  import java.util.NavigableMap;
28  import java.util.TreeMap;
29  import java.util.UUID;
30  
31  import org.apache.hadoop.hbase.classification.InterfaceAudience;
32  import org.apache.hadoop.hbase.classification.InterfaceStability;
33  import org.apache.hadoop.hbase.Cell;
34  import org.apache.hadoop.hbase.CellScannable;
35  import org.apache.hadoop.hbase.CellScanner;
36  import org.apache.hadoop.hbase.CellUtil;
37  import org.apache.hadoop.hbase.HConstants;
38  import org.apache.hadoop.hbase.KeyValue;
39  import org.apache.hadoop.hbase.KeyValueUtil;
40  import org.apache.hadoop.hbase.Tag;
41  import org.apache.hadoop.hbase.exceptions.DeserializationException;
42  import org.apache.hadoop.hbase.io.HeapSize;
43  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
44  import org.apache.hadoop.hbase.security.access.AccessControlConstants;
45  import org.apache.hadoop.hbase.security.access.Permission;
46  import org.apache.hadoop.hbase.security.visibility.CellVisibility;
47  import org.apache.hadoop.hbase.security.visibility.VisibilityConstants;
48  import org.apache.hadoop.hbase.util.Bytes;
49  import org.apache.hadoop.hbase.util.ClassSize;
50  
51  import com.google.common.collect.ArrayListMultimap;
52  import com.google.common.collect.ListMultimap;
53  import com.google.common.collect.Lists;
54  import com.google.common.io.ByteArrayDataInput;
55  import com.google.common.io.ByteArrayDataOutput;
56  import com.google.common.io.ByteStreams;
57  
58  @InterfaceAudience.Public
59  @InterfaceStability.Evolving
60  public abstract class Mutation extends OperationWithAttributes implements Row, CellScannable,
61      HeapSize {
62    public static final long MUTATION_OVERHEAD = ClassSize.align(
63        // This
64        ClassSize.OBJECT +
65        // row + OperationWithAttributes.attributes
66        2 * ClassSize.REFERENCE +
67        // Timestamp
68        1 * Bytes.SIZEOF_LONG +
69        // durability
70        ClassSize.REFERENCE +
71        // familyMap
72        ClassSize.REFERENCE +
73        // familyMap
74        ClassSize.TREEMAP);
75  
76    /**
77     * The attribute for storing the list of clusters that have consumed the change.
78     */
79    private static final String CONSUMED_CLUSTER_IDS = "_cs.id";
80  
81    /**
82     * The attribute for storing TTL for the result of the mutation.
83     */
84    private static final String OP_ATTRIBUTE_TTL = "_ttl";
85  
86    protected byte [] row = null;
87    protected long ts = HConstants.LATEST_TIMESTAMP;
88    protected Durability durability = Durability.USE_DEFAULT;
89  
90    // A Map sorted by column family.
91    protected NavigableMap<byte [], List<Cell>> familyMap =
92      new TreeMap<byte [], List<Cell>>(Bytes.BYTES_COMPARATOR);
93  
94    @Override
95    public CellScanner cellScanner() {
96      return CellUtil.createCellScanner(getFamilyCellMap());
97    }
98  
99    /**
100    * Creates an empty list if one doesn't exist for the given column family
101    * or else it returns the associated list of Cell objects.
102    *
103    * @param family column family
104    * @return a list of Cell objects, returns an empty list if one doesn't exist.
105    */
106   List<Cell> getCellList(byte[] family) {
107     List<Cell> list = this.familyMap.get(family);
108     if (list == null) {
109       list = new ArrayList<Cell>();
110     }
111     return list;
112   }
113 
114   /*
115    * Create a KeyValue with this objects row key and the Put identifier.
116    *
117    * @return a KeyValue with this objects row key and the Put identifier.
118    */
119   KeyValue createPutKeyValue(byte[] family, byte[] qualifier, long ts, byte[] value) {
120     return new KeyValue(this.row, family, qualifier, ts, KeyValue.Type.Put, value);
121   }
122 
123   /**
124    * Create a KeyValue with this objects row key and the Put identifier.
125    * @param family
126    * @param qualifier
127    * @param ts
128    * @param value
129    * @param tags - Specify the Tags as an Array {@link KeyValue.Tag}
130    * @return a KeyValue with this objects row key and the Put identifier.
131    */
132   KeyValue createPutKeyValue(byte[] family, byte[] qualifier, long ts, byte[] value, Tag[] tags) {
133     KeyValue kvWithTag = new KeyValue(this.row, family, qualifier, ts, value, tags);
134     return kvWithTag;
135   }
136 
137   /*
138    * Create a KeyValue with this objects row key and the Put identifier.
139    *
140    * @return a KeyValue with this objects row key and the Put identifier.
141    */
142   KeyValue createPutKeyValue(byte[] family, ByteBuffer qualifier, long ts, ByteBuffer value,
143                              Tag[] tags) {
144     return new KeyValue(this.row, 0, this.row == null ? 0 : this.row.length,
145         family, 0, family == null ? 0 : family.length,
146         qualifier, ts, KeyValue.Type.Put, value, tags != null ? Arrays.asList(tags) : null);
147   }
148 
149   /**
150    * Compile the column family (i.e. schema) information
151    * into a Map. Useful for parsing and aggregation by debugging,
152    * logging, and administration tools.
153    * @return Map
154    */
155   @Override
156   public Map<String, Object> getFingerprint() {
157     Map<String, Object> map = new HashMap<String, Object>();
158     List<String> families = new ArrayList<String>();
159     // ideally, we would also include table information, but that information
160     // is not stored in each Operation instance.
161     map.put("families", families);
162     for (Map.Entry<byte [], List<Cell>> entry : this.familyMap.entrySet()) {
163       families.add(Bytes.toStringBinary(entry.getKey()));
164     }
165     return map;
166   }
167 
168   /**
169    * Compile the details beyond the scope of getFingerprint (row, columns,
170    * timestamps, etc.) into a Map along with the fingerprinted information.
171    * Useful for debugging, logging, and administration tools.
172    * @param maxCols a limit on the number of columns output prior to truncation
173    * @return Map
174    */
175   @Override
176   public Map<String, Object> toMap(int maxCols) {
177     // we start with the fingerprint map and build on top of it.
178     Map<String, Object> map = getFingerprint();
179     // replace the fingerprint's simple list of families with a
180     // map from column families to lists of qualifiers and kv details
181     Map<String, List<Map<String, Object>>> columns =
182       new HashMap<String, List<Map<String, Object>>>();
183     map.put("families", columns);
184     map.put("row", Bytes.toStringBinary(this.row));
185     int colCount = 0;
186     // iterate through all column families affected
187     for (Map.Entry<byte [], List<Cell>> entry : this.familyMap.entrySet()) {
188       // map from this family to details for each cell affected within the family
189       List<Map<String, Object>> qualifierDetails = new ArrayList<Map<String, Object>>();
190       columns.put(Bytes.toStringBinary(entry.getKey()), qualifierDetails);
191       colCount += entry.getValue().size();
192       if (maxCols <= 0) {
193         continue;
194       }
195       // add details for each cell
196       for (Cell cell: entry.getValue()) {
197         if (--maxCols <= 0 ) {
198           continue;
199         }
200         // KeyValue v1 expectation.  Cast for now until we go all Cell all the time.
201         KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
202         Map<String, Object> kvMap = kv.toStringMap();
203         // row and family information are already available in the bigger map
204         kvMap.remove("row");
205         kvMap.remove("family");
206         qualifierDetails.add(kvMap);
207       }
208     }
209     map.put("totalColumns", colCount);
210     // add the id if set
211     if (getId() != null) {
212       map.put("id", getId());
213     }
214     // Add the TTL if set
215     // Long.MAX_VALUE is the default, and is interpreted to mean this attribute
216     // has not been set.
217     if (getTTL() != Long.MAX_VALUE) {
218       map.put("ttl", getTTL());
219     }
220     return map;
221   }
222 
223   /**
224    * @deprecated Use {@link #getDurability()} instead.
225    * @return true if edits should be applied to WAL, false if not
226    */
227   @Deprecated
228   public boolean getWriteToWAL() {
229     return this.durability == Durability.SKIP_WAL;
230   }
231 
232   /**
233    * Set whether this Delete should be written to the WAL or not.
234    * Not writing the WAL means you may lose edits on server crash.
235    * This method will reset any changes made via {@link #setDurability(Durability)}
236    * @param write true if edits should be written to WAL, false if not
237    * @deprecated Use {@link #setDurability(Durability)} instead.
238    */
239   @Deprecated
240   public void setWriteToWAL(boolean write) {
241     setDurability(write ? Durability.USE_DEFAULT : Durability.SKIP_WAL);
242   }
243 
244   /**
245    * Set the durability for this mutation
246    * @param d
247    */
248   public void setDurability(Durability d) {
249     this.durability = d;
250   }
251 
252   /** Get the current durability */
253   public Durability getDurability() {
254     return this.durability;
255   }
256 
257   /**
258    * Method for retrieving the put's familyMap
259    * @return familyMap
260    */
261   public NavigableMap<byte [], List<Cell>> getFamilyCellMap() {
262     return this.familyMap;
263   }
264 
265   /**
266    * Method for setting the put's familyMap
267    */
268   public void setFamilyCellMap(NavigableMap<byte [], List<Cell>> map) {
269     // TODO: Shut this down or move it up to be a Constructor.  Get new object rather than change
270     // this internal data member.
271     this.familyMap = map;
272   }
273 
274   /**
275    * Method for retrieving the put's familyMap that is deprecated and inefficient.
276    * @return the map
277    * @deprecated use {@link #getFamilyCellMap()} instead.
278    */
279   @Deprecated
280   public NavigableMap<byte [], List<KeyValue>> getFamilyMap() {
281     TreeMap<byte[], List<KeyValue>> fm =
282         new TreeMap<byte[], List<KeyValue>>(Bytes.BYTES_COMPARATOR);
283     for (Map.Entry<byte[], List<Cell>> e : familyMap.entrySet()) {
284       List<KeyValue> kvl = new ArrayList<KeyValue>(e.getValue().size());
285       for (Cell c : e.getValue()) {
286         kvl.add(KeyValueUtil.ensureKeyValue(c));
287       }
288       fm.put(e.getKey(), kvl);
289     }
290     return fm;
291   }
292 
293   /**
294    * Method for setting the put's familyMap that is deprecated and inefficient.
295    * @deprecated use {@link #setFamilyCellMap(NavigableMap)} instead.
296    */
297   @Deprecated
298   public void setFamilyMap(NavigableMap<byte [], List<KeyValue>> map) {
299     TreeMap<byte[], List<Cell>> fm = new TreeMap<byte[], List<Cell>>(Bytes.BYTES_COMPARATOR);
300     for (Map.Entry<byte[], List<KeyValue>> e : map.entrySet()) {
301       fm.put(e.getKey(), Lists.<Cell>newArrayList(e.getValue()));
302     }
303     this.familyMap = fm;
304   }
305 
306   /**
307    * Method to check if the familyMap is empty
308    * @return true if empty, false otherwise
309    */
310   public boolean isEmpty() {
311     return familyMap.isEmpty();
312   }
313 
314   /**
315    * Method for retrieving the delete's row
316    * @return row
317    */
318   @Override
319   public byte [] getRow() {
320     return this.row;
321   }
322 
323   @Override
324   public int compareTo(final Row d) {
325     return Bytes.compareTo(this.getRow(), d.getRow());
326   }
327 
328   /**
329    * Method for retrieving the timestamp
330    * @return timestamp
331    */
332   public long getTimeStamp() {
333     return this.ts;
334   }
335 
336   /**
337    * Marks that the clusters with the given clusterIds have consumed the mutation
338    * @param clusterIds of the clusters that have consumed the mutation
339    */
340   public void setClusterIds(List<UUID> clusterIds) {
341     ByteArrayDataOutput out = ByteStreams.newDataOutput();
342     out.writeInt(clusterIds.size());
343     for (UUID clusterId : clusterIds) {
344       out.writeLong(clusterId.getMostSignificantBits());
345       out.writeLong(clusterId.getLeastSignificantBits());
346     }
347     setAttribute(CONSUMED_CLUSTER_IDS, out.toByteArray());
348   }
349 
350   /**
351    * @return the set of clusterIds that have consumed the mutation
352    */
353   public List<UUID> getClusterIds() {
354     List<UUID> clusterIds = new ArrayList<UUID>();
355     byte[] bytes = getAttribute(CONSUMED_CLUSTER_IDS);
356     if(bytes != null) {
357       ByteArrayDataInput in = ByteStreams.newDataInput(bytes);
358       int numClusters = in.readInt();
359       for(int i=0; i<numClusters; i++){
360         clusterIds.add(new UUID(in.readLong(), in.readLong()));
361       }
362     }
363     return clusterIds;
364   }
365 
366   /**
367    * Sets the visibility expression associated with cells in this Mutation.
368    * It is illegal to set <code>CellVisibility</code> on <code>Delete</code> mutation.
369    * @param expression
370    */
371   @InterfaceStability.Unstable
372   public void setCellVisibility(CellVisibility expression) {
373     this.setAttribute(VisibilityConstants.VISIBILITY_LABELS_ATTR_KEY, ProtobufUtil
374         .toCellVisibility(expression).toByteArray());
375   }
376 
377   /**
378    * @return CellVisibility associated with cells in this Mutation.
379    * @throws DeserializationException
380    */
381   @InterfaceStability.Unstable
382   public CellVisibility getCellVisibility() throws DeserializationException {
383     byte[] cellVisibilityBytes = this.getAttribute(VisibilityConstants.VISIBILITY_LABELS_ATTR_KEY);
384     if (cellVisibilityBytes == null) return null;
385     return ProtobufUtil.toCellVisibility(cellVisibilityBytes);
386   }
387 
388   /**
389    * Number of KeyValues carried by this Mutation.
390    * @return the total number of KeyValues
391    */
392   public int size() {
393     int size = 0;
394     for (List<Cell> cells : this.familyMap.values()) {
395       size += cells.size();
396     }
397     return size;
398   }
399 
400   /**
401    * @return the number of different families
402    */
403   public int numFamilies() {
404     return familyMap.size();
405   }
406 
407   /**
408    * @return Calculate what Mutation adds to class heap size.
409    */
410   @Override
411   public long heapSize() {
412     long heapsize = MUTATION_OVERHEAD;
413     // Adding row
414     heapsize += ClassSize.align(ClassSize.ARRAY + this.row.length);
415 
416     // Adding map overhead
417     heapsize +=
418       ClassSize.align(this.familyMap.size() * ClassSize.MAP_ENTRY);
419     for(Map.Entry<byte [], List<Cell>> entry : this.familyMap.entrySet()) {
420       //Adding key overhead
421       heapsize +=
422         ClassSize.align(ClassSize.ARRAY + entry.getKey().length);
423 
424       //This part is kinds tricky since the JVM can reuse references if you
425       //store the same value, but have a good match with SizeOf at the moment
426       //Adding value overhead
427       heapsize += ClassSize.align(ClassSize.ARRAYLIST);
428       int size = entry.getValue().size();
429       heapsize += ClassSize.align(ClassSize.ARRAY +
430           size * ClassSize.REFERENCE);
431 
432       for(Cell cell : entry.getValue()) {
433         KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
434         heapsize += kv.heapSize();
435       }
436     }
437     heapsize += getAttributeSize();
438     heapsize += extraHeapSize();
439     return ClassSize.align(heapsize);
440   }
441 
442   /**
443    * @return The serialized ACL for this operation, or null if none
444    */
445   @InterfaceStability.Unstable
446   public byte[] getACL() {
447     return getAttribute(AccessControlConstants.OP_ATTRIBUTE_ACL);
448   }
449 
450   /**
451    * @param user User short name
452    * @param perms Permissions for the user
453    */
454   @InterfaceStability.Unstable
455   public void setACL(String user, Permission perms) {
456     setAttribute(AccessControlConstants.OP_ATTRIBUTE_ACL,
457       ProtobufUtil.toUsersAndPermissions(user, perms).toByteArray());
458   }
459 
460   /**
461    * @param perms A map of permissions for a user or users
462    */
463   @InterfaceStability.Unstable
464   public void setACL(Map<String, Permission> perms) {
465     ListMultimap<String, Permission> permMap = ArrayListMultimap.create();
466     for (Map.Entry<String, Permission> entry : perms.entrySet()) {
467       permMap.put(entry.getKey(), entry.getValue());
468     }
469     setAttribute(AccessControlConstants.OP_ATTRIBUTE_ACL,
470       ProtobufUtil.toUsersAndPermissions(permMap).toByteArray());
471   }
472 
473   /**
474    * @deprecated No effect
475    */
476   @Deprecated
477   public boolean getACLStrategy() {
478     return false;
479   }
480 
481   /**
482    * @deprecated No effect
483    */
484   @Deprecated
485   public void setACLStrategy(boolean cellFirstStrategy) {
486   }
487 
488   /**
489    * Return the TTL requested for the result of the mutation, in milliseconds.
490    * @return the TTL requested for the result of the mutation, in milliseconds,
491    * or Long.MAX_VALUE if unset
492    */
493   @InterfaceStability.Unstable
494   public long getTTL() {
495     byte[] ttlBytes = getAttribute(OP_ATTRIBUTE_TTL);
496     if (ttlBytes != null) {
497       return Bytes.toLong(ttlBytes);
498     }
499     return Long.MAX_VALUE;
500   }
501 
502   /**
503    * Set the TTL desired for the result of the mutation, in milliseconds.
504    * @param ttl the TTL desired for the result of the mutation, in milliseconds
505    * @return this
506    */
507   @InterfaceStability.Unstable
508   public Mutation setTTL(long ttl) {
509     setAttribute(OP_ATTRIBUTE_TTL, Bytes.toBytes(ttl));
510     return this;
511   }
512 
513   /**
514    * Subclasses should override this method to add the heap size of their own fields.
515    * @return the heap size to add (will be aligned).
516    */
517   protected long extraHeapSize(){
518     return 0L;
519   }
520 
521   /**
522    * @param row Row to check
523    * @throws IllegalArgumentException Thrown if <code>row</code> is empty or null or
524    * &gt; {@link HConstants#MAX_ROW_LENGTH}
525    * @return <code>row</code>
526    */
527   static byte [] checkRow(final byte [] row) {
528     return checkRow(row, 0, row == null? 0: row.length);
529   }
530 
531   /**
532    * @param row Row to check
533    * @param offset
534    * @param length
535    * @throws IllegalArgumentException Thrown if <code>row</code> is empty or null or
536    * &gt; {@link HConstants#MAX_ROW_LENGTH}
537    * @return <code>row</code>
538    */
539   static byte [] checkRow(final byte [] row, final int offset, final int length) {
540     if (row == null) {
541       throw new IllegalArgumentException("Row buffer is null");
542     }
543     if (length == 0) {
544       throw new IllegalArgumentException("Row length is 0");
545     }
546     if (length > HConstants.MAX_ROW_LENGTH) {
547       throw new IllegalArgumentException("Row length " + length + " is > " +
548         HConstants.MAX_ROW_LENGTH);
549     }
550     return row;
551   }
552 
553   static void checkRow(ByteBuffer row) {
554     if (row == null) {
555       throw new IllegalArgumentException("Row buffer is null");
556     }
557     if (row.remaining() == 0) {
558       throw new IllegalArgumentException("Row length is 0");
559     }
560     if (row.remaining() > HConstants.MAX_ROW_LENGTH) {
561       throw new IllegalArgumentException("Row length " + row.remaining() + " is > " +
562           HConstants.MAX_ROW_LENGTH);
563     }
564   }
565 }