View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.client;
20  
21  import java.nio.ByteBuffer;
22  import java.util.ArrayList;
23  import java.util.Arrays;
24  import java.util.HashMap;
25  import java.util.List;
26  import java.util.Map;
27  import java.util.NavigableMap;
28  import java.util.TreeMap;
29  import java.util.UUID;
30  
31  import org.apache.hadoop.hbase.classification.InterfaceAudience;
32  import org.apache.hadoop.hbase.classification.InterfaceStability;
33  import org.apache.hadoop.hbase.Cell;
34  import org.apache.hadoop.hbase.CellScannable;
35  import org.apache.hadoop.hbase.CellScanner;
36  import org.apache.hadoop.hbase.CellUtil;
37  import org.apache.hadoop.hbase.HConstants;
38  import org.apache.hadoop.hbase.KeyValue;
39  import org.apache.hadoop.hbase.KeyValueUtil;
40  import org.apache.hadoop.hbase.Tag;
41  import org.apache.hadoop.hbase.exceptions.DeserializationException;
42  import org.apache.hadoop.hbase.io.HeapSize;
43  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
44  import org.apache.hadoop.hbase.security.access.AccessControlConstants;
45  import org.apache.hadoop.hbase.security.access.Permission;
46  import org.apache.hadoop.hbase.security.visibility.CellVisibility;
47  import org.apache.hadoop.hbase.security.visibility.VisibilityConstants;
48  import org.apache.hadoop.hbase.util.Bytes;
49  import org.apache.hadoop.hbase.util.ClassSize;
50  
51  import com.google.common.collect.ArrayListMultimap;
52  import com.google.common.collect.ListMultimap;
53  import com.google.common.collect.Lists;
54  import com.google.common.io.ByteArrayDataInput;
55  import com.google.common.io.ByteArrayDataOutput;
56  import com.google.common.io.ByteStreams;
57  
58  @InterfaceAudience.Public
59  @InterfaceStability.Evolving
60  public abstract class Mutation extends OperationWithAttributes implements Row, CellScannable,
61      HeapSize {
62    public static final long MUTATION_OVERHEAD = ClassSize.align(
63        // This
64        ClassSize.OBJECT +
65        // row + OperationWithAttributes.attributes
66        2 * ClassSize.REFERENCE +
67        // Timestamp
68        1 * Bytes.SIZEOF_LONG +
69        // durability
70        ClassSize.REFERENCE +
71        // familyMap
72        ClassSize.REFERENCE +
73        // familyMap
74        ClassSize.TREEMAP);
75  
76    /**
77     * The attribute for storing the list of clusters that have consumed the change.
78     */
79    private static final String CONSUMED_CLUSTER_IDS = "_cs.id";
80  
81    /**
82     * The attribute for storing TTL for the result of the mutation.
83     */
84    private static final String OP_ATTRIBUTE_TTL = "_ttl";
85  
86    /**
87     * @deprecated this field is private as of HBase 2.0.
88     */
89    @Deprecated
90    protected static final String RETURN_RESULTS = "_rr_";
91  
92    protected byte [] row = null;
93    protected long ts = HConstants.LATEST_TIMESTAMP;
94    protected Durability durability = Durability.USE_DEFAULT;
95  
96    // A Map sorted by column family.
97    protected NavigableMap<byte [], List<Cell>> familyMap =
98      new TreeMap<byte [], List<Cell>>(Bytes.BYTES_COMPARATOR);
99  
100   @Override
101   public CellScanner cellScanner() {
102     return CellUtil.createCellScanner(getFamilyCellMap());
103   }
104 
105   /**
106    * Creates an empty list if one doesn't exist for the given column family
107    * or else it returns the associated list of Cell objects.
108    *
109    * @param family column family
110    * @return a list of Cell objects, returns an empty list if one doesn't exist.
111    */
112   List<Cell> getCellList(byte[] family) {
113     List<Cell> list = this.familyMap.get(family);
114     if (list == null) {
115       list = new ArrayList<Cell>();
116     }
117     return list;
118   }
119 
120   /*
121    * Create a KeyValue with this objects row key and the Put identifier.
122    *
123    * @return a KeyValue with this objects row key and the Put identifier.
124    */
125   KeyValue createPutKeyValue(byte[] family, byte[] qualifier, long ts, byte[] value) {
126     return new KeyValue(this.row, family, qualifier, ts, KeyValue.Type.Put, value);
127   }
128 
129   /**
130    * Create a KeyValue with this objects row key and the Put identifier.
131    * @param family
132    * @param qualifier
133    * @param ts
134    * @param value
135    * @param tags - Specify the Tags as an Array {@link KeyValue.Tag}
136    * @return a KeyValue with this objects row key and the Put identifier.
137    */
138   KeyValue createPutKeyValue(byte[] family, byte[] qualifier, long ts, byte[] value, Tag[] tags) {
139     KeyValue kvWithTag = new KeyValue(this.row, family, qualifier, ts, value, tags);
140     return kvWithTag;
141   }
142 
143   /*
144    * Create a KeyValue with this objects row key and the Put identifier.
145    *
146    * @return a KeyValue with this objects row key and the Put identifier.
147    */
148   KeyValue createPutKeyValue(byte[] family, ByteBuffer qualifier, long ts, ByteBuffer value,
149                              Tag[] tags) {
150     return new KeyValue(this.row, 0, this.row == null ? 0 : this.row.length,
151         family, 0, family == null ? 0 : family.length,
152         qualifier, ts, KeyValue.Type.Put, value, tags != null ? Arrays.asList(tags) : null);
153   }
154 
155   /**
156    * Compile the column family (i.e. schema) information
157    * into a Map. Useful for parsing and aggregation by debugging,
158    * logging, and administration tools.
159    * @return Map
160    */
161   @Override
162   public Map<String, Object> getFingerprint() {
163     Map<String, Object> map = new HashMap<String, Object>();
164     List<String> families = new ArrayList<String>();
165     // ideally, we would also include table information, but that information
166     // is not stored in each Operation instance.
167     map.put("families", families);
168     for (Map.Entry<byte [], List<Cell>> entry : this.familyMap.entrySet()) {
169       families.add(Bytes.toStringBinary(entry.getKey()));
170     }
171     return map;
172   }
173 
174   /**
175    * Compile the details beyond the scope of getFingerprint (row, columns,
176    * timestamps, etc.) into a Map along with the fingerprinted information.
177    * Useful for debugging, logging, and administration tools.
178    * @param maxCols a limit on the number of columns output prior to truncation
179    * @return Map
180    */
181   @Override
182   public Map<String, Object> toMap(int maxCols) {
183     // we start with the fingerprint map and build on top of it.
184     Map<String, Object> map = getFingerprint();
185     // replace the fingerprint's simple list of families with a
186     // map from column families to lists of qualifiers and kv details
187     Map<String, List<Map<String, Object>>> columns =
188       new HashMap<String, List<Map<String, Object>>>();
189     map.put("families", columns);
190     map.put("row", Bytes.toStringBinary(this.row));
191     int colCount = 0;
192     // iterate through all column families affected
193     for (Map.Entry<byte [], List<Cell>> entry : this.familyMap.entrySet()) {
194       // map from this family to details for each cell affected within the family
195       List<Map<String, Object>> qualifierDetails = new ArrayList<Map<String, Object>>();
196       columns.put(Bytes.toStringBinary(entry.getKey()), qualifierDetails);
197       colCount += entry.getValue().size();
198       if (maxCols <= 0) {
199         continue;
200       }
201       // add details for each cell
202       for (Cell cell: entry.getValue()) {
203         if (--maxCols <= 0 ) {
204           continue;
205         }
206         Map<String, Object> cellMap = cellToStringMap(cell);
207         qualifierDetails.add(cellMap);
208       }
209     }
210     map.put("totalColumns", colCount);
211     // add the id if set
212     if (getId() != null) {
213       map.put("id", getId());
214     }
215     // Add the TTL if set
216     // Long.MAX_VALUE is the default, and is interpreted to mean this attribute
217     // has not been set.
218     if (getTTL() != Long.MAX_VALUE) {
219       map.put("ttl", getTTL());
220     }
221     return map;
222   }
223 
224   private static Map<String, Object> cellToStringMap(Cell c) {
225     Map<String, Object> stringMap = new HashMap<String, Object>();
226     stringMap.put("qualifier", Bytes.toStringBinary(c.getQualifierArray(), c.getQualifierOffset(),
227                 c.getQualifierLength()));
228     stringMap.put("timestamp", c.getTimestamp());
229     stringMap.put("vlen", c.getValueLength());
230     List<Tag> tags = Tag.asList(c.getTagsArray(), c.getTagsOffset(), c.getTagsLength());
231     if (tags != null) {
232       List<String> tagsString = new ArrayList<String>();
233       for (Tag t : tags) {
234         tagsString.add((t.getType()) + ":" + Bytes.toStringBinary(t.getValue()));
235       }
236       stringMap.put("tag", tagsString);
237     }
238     return stringMap;
239   }
240 
241   /**
242    * @deprecated Use {@link #getDurability()} instead.
243    * @return true if edits should be applied to WAL, false if not
244    */
245   @Deprecated
246   public boolean getWriteToWAL() {
247     return this.durability != Durability.SKIP_WAL;
248   }
249 
250   /**
251    * Set whether this Delete should be written to the WAL or not.
252    * Not writing the WAL means you may lose edits on server crash.
253    * This method will reset any changes made via {@link #setDurability(Durability)}
254    * @param write true if edits should be written to WAL, false if not
255    * @deprecated Use {@link #setDurability(Durability)} instead.
256    */
257   @Deprecated
258   public Mutation setWriteToWAL(boolean write) {
259     setDurability(write ? Durability.USE_DEFAULT : Durability.SKIP_WAL);
260     return this;
261   }
262 
263   /**
264    * Set the durability for this mutation
265    * @param d
266    */
267   public Mutation setDurability(Durability d) {
268     this.durability = d;
269     return this;
270   }
271 
272   /** Get the current durability */
273   public Durability getDurability() {
274     return this.durability;
275   }
276 
277   /**
278    * Method for retrieving the put's familyMap
279    * @return familyMap
280    */
281   public NavigableMap<byte [], List<Cell>> getFamilyCellMap() {
282     return this.familyMap;
283   }
284 
285   /**
286    * Method for setting the put's familyMap
287    */
288   public Mutation setFamilyCellMap(NavigableMap<byte [], List<Cell>> map) {
289     // TODO: Shut this down or move it up to be a Constructor.  Get new object rather than change
290     // this internal data member.
291     this.familyMap = map;
292     return this;
293   }
294 
295   /**
296    * Method for retrieving the put's familyMap that is deprecated and inefficient.
297    * @return the map
298    * @deprecated use {@link #getFamilyCellMap()} instead.
299    */
300   @Deprecated
301   public NavigableMap<byte [], List<KeyValue>> getFamilyMap() {
302     TreeMap<byte[], List<KeyValue>> fm =
303         new TreeMap<byte[], List<KeyValue>>(Bytes.BYTES_COMPARATOR);
304     for (Map.Entry<byte[], List<Cell>> e : familyMap.entrySet()) {
305       List<KeyValue> kvl = new ArrayList<KeyValue>(e.getValue().size());
306       for (Cell c : e.getValue()) {
307         kvl.add(KeyValueUtil.ensureKeyValue(c));
308       }
309       fm.put(e.getKey(), kvl);
310     }
311     return fm;
312   }
313 
314   /**
315    * Method for setting the put's familyMap that is deprecated and inefficient.
316    * @deprecated use {@link #setFamilyCellMap(NavigableMap)} instead.
317    */
318   @Deprecated
319   public Mutation setFamilyMap(NavigableMap<byte [], List<KeyValue>> map) {
320     TreeMap<byte[], List<Cell>> fm = new TreeMap<byte[], List<Cell>>(Bytes.BYTES_COMPARATOR);
321     for (Map.Entry<byte[], List<KeyValue>> e : map.entrySet()) {
322       fm.put(e.getKey(), Lists.<Cell>newArrayList(e.getValue()));
323     }
324     this.familyMap = fm;
325     return this;
326   }
327 
328   /**
329    * Method to check if the familyMap is empty
330    * @return true if empty, false otherwise
331    */
332   public boolean isEmpty() {
333     return familyMap.isEmpty();
334   }
335 
336   /**
337    * Method for retrieving the delete's row
338    * @return row
339    */
340   @Override
341   public byte [] getRow() {
342     return this.row;
343   }
344 
345   @Override
346   public int compareTo(final Row d) {
347     return Bytes.compareTo(this.getRow(), d.getRow());
348   }
349 
350   /**
351    * Method for retrieving the timestamp
352    * @return timestamp
353    */
354   public long getTimeStamp() {
355     return this.ts;
356   }
357 
358   /**
359    * Marks that the clusters with the given clusterIds have consumed the mutation
360    * @param clusterIds of the clusters that have consumed the mutation
361    */
362   public Mutation setClusterIds(List<UUID> clusterIds) {
363     ByteArrayDataOutput out = ByteStreams.newDataOutput();
364     out.writeInt(clusterIds.size());
365     for (UUID clusterId : clusterIds) {
366       out.writeLong(clusterId.getMostSignificantBits());
367       out.writeLong(clusterId.getLeastSignificantBits());
368     }
369     setAttribute(CONSUMED_CLUSTER_IDS, out.toByteArray());
370     return this;
371   }
372 
373   /**
374    * @return the set of clusterIds that have consumed the mutation
375    */
376   public List<UUID> getClusterIds() {
377     List<UUID> clusterIds = new ArrayList<UUID>();
378     byte[] bytes = getAttribute(CONSUMED_CLUSTER_IDS);
379     if(bytes != null) {
380       ByteArrayDataInput in = ByteStreams.newDataInput(bytes);
381       int numClusters = in.readInt();
382       for(int i=0; i<numClusters; i++){
383         clusterIds.add(new UUID(in.readLong(), in.readLong()));
384       }
385     }
386     return clusterIds;
387   }
388 
389   /**
390    * Sets the visibility expression associated with cells in this Mutation.
391    * It is illegal to set <code>CellVisibility</code> on <code>Delete</code> mutation.
392    * @param expression
393    */
394   public Mutation setCellVisibility(CellVisibility expression) {
395     this.setAttribute(VisibilityConstants.VISIBILITY_LABELS_ATTR_KEY, ProtobufUtil
396         .toCellVisibility(expression).toByteArray());
397     return this;
398   }
399 
400   /**
401    * @return CellVisibility associated with cells in this Mutation.
402    * @throws DeserializationException
403    */
404   public CellVisibility getCellVisibility() throws DeserializationException {
405     byte[] cellVisibilityBytes = this.getAttribute(VisibilityConstants.VISIBILITY_LABELS_ATTR_KEY);
406     if (cellVisibilityBytes == null) return null;
407     return ProtobufUtil.toCellVisibility(cellVisibilityBytes);
408   }
409 
410   /**
411    * Number of KeyValues carried by this Mutation.
412    * @return the total number of KeyValues
413    */
414   public int size() {
415     int size = 0;
416     for (List<Cell> cells : this.familyMap.values()) {
417       size += cells.size();
418     }
419     return size;
420   }
421 
422   /**
423    * @return the number of different families
424    */
425   public int numFamilies() {
426     return familyMap.size();
427   }
428 
429   /**
430    * @return Calculate what Mutation adds to class heap size.
431    */
432   @Override
433   public long heapSize() {
434     long heapsize = MUTATION_OVERHEAD;
435     // Adding row
436     heapsize += ClassSize.align(ClassSize.ARRAY + this.row.length);
437 
438     // Adding map overhead
439     heapsize +=
440       ClassSize.align(this.familyMap.size() * ClassSize.MAP_ENTRY);
441     for(Map.Entry<byte [], List<Cell>> entry : this.familyMap.entrySet()) {
442       //Adding key overhead
443       heapsize +=
444         ClassSize.align(ClassSize.ARRAY + entry.getKey().length);
445 
446       //This part is kinds tricky since the JVM can reuse references if you
447       //store the same value, but have a good match with SizeOf at the moment
448       //Adding value overhead
449       heapsize += ClassSize.align(ClassSize.ARRAYLIST);
450       int size = entry.getValue().size();
451       heapsize += ClassSize.align(ClassSize.ARRAY +
452           size * ClassSize.REFERENCE);
453 
454       for(Cell cell : entry.getValue()) {
455         heapsize += CellUtil.estimatedHeapSizeOf(cell);
456       }
457     }
458     heapsize += getAttributeSize();
459     heapsize += extraHeapSize();
460     return ClassSize.align(heapsize);
461   }
462 
463   /**
464    * @return The serialized ACL for this operation, or null if none
465    */
466   public byte[] getACL() {
467     return getAttribute(AccessControlConstants.OP_ATTRIBUTE_ACL);
468   }
469 
470   /**
471    * @param user User short name
472    * @param perms Permissions for the user
473    */
474   public Mutation setACL(String user, Permission perms) {
475     setAttribute(AccessControlConstants.OP_ATTRIBUTE_ACL,
476       ProtobufUtil.toUsersAndPermissions(user, perms).toByteArray());
477     return this;
478   }
479 
480   /**
481    * @param perms A map of permissions for a user or users
482    */
483   public Mutation setACL(Map<String, Permission> perms) {
484     ListMultimap<String, Permission> permMap = ArrayListMultimap.create();
485     for (Map.Entry<String, Permission> entry : perms.entrySet()) {
486       permMap.put(entry.getKey(), entry.getValue());
487     }
488     setAttribute(AccessControlConstants.OP_ATTRIBUTE_ACL,
489       ProtobufUtil.toUsersAndPermissions(permMap).toByteArray());
490     return this;
491   }
492 
493   /**
494    * Return the TTL requested for the result of the mutation, in milliseconds.
495    * @return the TTL requested for the result of the mutation, in milliseconds,
496    * or Long.MAX_VALUE if unset
497    */
498   public long getTTL() {
499     byte[] ttlBytes = getAttribute(OP_ATTRIBUTE_TTL);
500     if (ttlBytes != null) {
501       return Bytes.toLong(ttlBytes);
502     }
503     return Long.MAX_VALUE;
504   }
505 
506   /**
507    * Set the TTL desired for the result of the mutation, in milliseconds.
508    * @param ttl the TTL desired for the result of the mutation, in milliseconds
509    * @return this
510    */
511   public Mutation setTTL(long ttl) {
512     setAttribute(OP_ATTRIBUTE_TTL, Bytes.toBytes(ttl));
513     return this;
514   }
515 
516   /**
517    * @return current value for returnResults
518    */
519   // Used by Increment and Append only.
520   @InterfaceAudience.Private
521   protected boolean isReturnResults() {
522     byte[] v = getAttribute(RETURN_RESULTS);
523     return v == null ? true : Bytes.toBoolean(v);
524   }
525 
526   @InterfaceAudience.Private
527   // Used by Increment and Append only.
528   protected Mutation setReturnResults(boolean returnResults) {
529     setAttribute(RETURN_RESULTS, Bytes.toBytes(returnResults));
530     return this;
531   }
532 
533   /**
534    * Subclasses should override this method to add the heap size of their own fields.
535    * @return the heap size to add (will be aligned).
536    */
537   protected long extraHeapSize(){
538     return 0L;
539   }
540 
541 
542   /**
543    * @param row Row to check
544    * @throws IllegalArgumentException Thrown if <code>row</code> is empty or null or
545    * &gt; {@link HConstants#MAX_ROW_LENGTH}
546    * @return <code>row</code>
547    */
548   static byte [] checkRow(final byte [] row) {
549     return checkRow(row, 0, row == null? 0: row.length);
550   }
551 
552   /**
553    * @param row Row to check
554    * @param offset
555    * @param length
556    * @throws IllegalArgumentException Thrown if <code>row</code> is empty or null or
557    * &gt; {@link HConstants#MAX_ROW_LENGTH}
558    * @return <code>row</code>
559    */
560   static byte [] checkRow(final byte [] row, final int offset, final int length) {
561     if (row == null) {
562       throw new IllegalArgumentException("Row buffer is null");
563     }
564     if (length == 0) {
565       throw new IllegalArgumentException("Row length is 0");
566     }
567     if (length > HConstants.MAX_ROW_LENGTH) {
568       throw new IllegalArgumentException("Row length " + length + " is > " +
569         HConstants.MAX_ROW_LENGTH);
570     }
571     return row;
572   }
573 
574   static void checkRow(ByteBuffer row) {
575     if (row == null) {
576       throw new IllegalArgumentException("Row buffer is null");
577     }
578     if (row.remaining() == 0) {
579       throw new IllegalArgumentException("Row length is 0");
580     }
581     if (row.remaining() > HConstants.MAX_ROW_LENGTH) {
582       throw new IllegalArgumentException("Row length " + row.remaining() + " is > " +
583           HConstants.MAX_ROW_LENGTH);
584     }
585   }
586 }