View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.client;
20  
21  import java.nio.ByteBuffer;
22  import java.util.ArrayList;
23  import java.util.Arrays;
24  import java.util.HashMap;
25  import java.util.List;
26  import java.util.Map;
27  import java.util.NavigableMap;
28  import java.util.TreeMap;
29  import java.util.UUID;
30  
31  import org.apache.hadoop.hbase.Cell;
32  import org.apache.hadoop.hbase.CellScannable;
33  import org.apache.hadoop.hbase.CellScanner;
34  import org.apache.hadoop.hbase.CellUtil;
35  import org.apache.hadoop.hbase.HConstants;
36  import org.apache.hadoop.hbase.KeyValue;
37  import org.apache.hadoop.hbase.Tag;
38  import org.apache.hadoop.hbase.TagUtil;
39  import org.apache.hadoop.hbase.classification.InterfaceAudience;
40  import org.apache.hadoop.hbase.classification.InterfaceStability;
41  import org.apache.hadoop.hbase.exceptions.DeserializationException;
42  import org.apache.hadoop.hbase.io.HeapSize;
43  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
44  import org.apache.hadoop.hbase.security.access.AccessControlConstants;
45  import org.apache.hadoop.hbase.security.access.Permission;
46  import org.apache.hadoop.hbase.security.visibility.CellVisibility;
47  import org.apache.hadoop.hbase.security.visibility.VisibilityConstants;
48  import org.apache.hadoop.hbase.util.Bytes;
49  import org.apache.hadoop.hbase.util.ClassSize;
50  
51  import com.google.common.collect.ArrayListMultimap;
52  import com.google.common.collect.ListMultimap;
53  import com.google.common.io.ByteArrayDataInput;
54  import com.google.common.io.ByteArrayDataOutput;
55  import com.google.common.io.ByteStreams;
56  
57  @InterfaceAudience.Public
58  @InterfaceStability.Evolving
59  public abstract class Mutation extends OperationWithAttributes implements Row, CellScannable,
60      HeapSize {
61    public static final long MUTATION_OVERHEAD = ClassSize.align(
62        // This
63        ClassSize.OBJECT +
64        // row + OperationWithAttributes.attributes
65        2 * ClassSize.REFERENCE +
66        // Timestamp
67        1 * Bytes.SIZEOF_LONG +
68        // durability
69        ClassSize.REFERENCE +
70        // familyMap
71        ClassSize.REFERENCE +
72        // familyMap
73        ClassSize.TREEMAP);
74  
75    /**
76     * The attribute for storing the list of clusters that have consumed the change.
77     */
78    private static final String CONSUMED_CLUSTER_IDS = "_cs.id";
79  
80    /**
81     * The attribute for storing TTL for the result of the mutation.
82     */
83    private static final String OP_ATTRIBUTE_TTL = "_ttl";
84  
85    private static final String RETURN_RESULTS = "_rr_";
86  
87    protected byte [] row = null;
88    protected long ts = HConstants.LATEST_TIMESTAMP;
89    protected Durability durability = Durability.USE_DEFAULT;
90  
91    // A Map sorted by column family.
92    protected NavigableMap<byte [], List<Cell>> familyMap =
93      new TreeMap<byte [], List<Cell>>(Bytes.BYTES_COMPARATOR);
94  
95    @Override
96    public CellScanner cellScanner() {
97      return CellUtil.createCellScanner(getFamilyCellMap());
98    }
99  
100   /**
101    * Creates an empty list if one doesn't exist for the given column family
102    * or else it returns the associated list of Cell objects.
103    *
104    * @param family column family
105    * @return a list of Cell objects, returns an empty list if one doesn't exist.
106    */
107   List<Cell> getCellList(byte[] family) {
108     List<Cell> list = this.familyMap.get(family);
109     if (list == null) {
110       list = new ArrayList<Cell>();
111     }
112     return list;
113   }
114 
115   /*
116    * Create a KeyValue with this objects row key and the Put identifier.
117    *
118    * @return a KeyValue with this objects row key and the Put identifier.
119    */
120   KeyValue createPutKeyValue(byte[] family, byte[] qualifier, long ts, byte[] value) {
121     return new KeyValue(this.row, family, qualifier, ts, KeyValue.Type.Put, value);
122   }
123 
124   /**
125    * Create a KeyValue with this objects row key and the Put identifier.
126    * @param family
127    * @param qualifier
128    * @param ts
129    * @param value
130    * @param tags - Specify the Tags as an Array
131    * @return a KeyValue with this objects row key and the Put identifier.
132    */
133   KeyValue createPutKeyValue(byte[] family, byte[] qualifier, long ts, byte[] value, Tag[] tags) {
134     KeyValue kvWithTag = new KeyValue(this.row, family, qualifier, ts, value, tags);
135     return kvWithTag;
136   }
137 
138   /*
139    * Create a KeyValue with this objects row key and the Put identifier.
140    *
141    * @return a KeyValue with this objects row key and the Put identifier.
142    */
143   KeyValue createPutKeyValue(byte[] family, ByteBuffer qualifier, long ts, ByteBuffer value,
144       Tag[] tags) {
145     return new KeyValue(this.row, 0, this.row == null ? 0 : this.row.length,
146         family, 0, family == null ? 0 : family.length,
147         qualifier, ts, KeyValue.Type.Put, value, tags != null ? Arrays.asList(tags) : null);
148   }
149 
150   /**
151    * Compile the column family (i.e. schema) information
152    * into a Map. Useful for parsing and aggregation by debugging,
153    * logging, and administration tools.
154    * @return Map
155    */
156   @Override
157   public Map<String, Object> getFingerprint() {
158     Map<String, Object> map = new HashMap<String, Object>();
159     List<String> families = new ArrayList<String>();
160     // ideally, we would also include table information, but that information
161     // is not stored in each Operation instance.
162     map.put("families", families);
163     for (Map.Entry<byte [], List<Cell>> entry : this.familyMap.entrySet()) {
164       families.add(Bytes.toStringBinary(entry.getKey()));
165     }
166     return map;
167   }
168 
169   /**
170    * Compile the details beyond the scope of getFingerprint (row, columns,
171    * timestamps, etc.) into a Map along with the fingerprinted information.
172    * Useful for debugging, logging, and administration tools.
173    * @param maxCols a limit on the number of columns output prior to truncation
174    * @return Map
175    */
176   @Override
177   public Map<String, Object> toMap(int maxCols) {
178     // we start with the fingerprint map and build on top of it.
179     Map<String, Object> map = getFingerprint();
180     // replace the fingerprint's simple list of families with a
181     // map from column families to lists of qualifiers and kv details
182     Map<String, List<Map<String, Object>>> columns =
183       new HashMap<String, List<Map<String, Object>>>();
184     map.put("families", columns);
185     map.put("row", Bytes.toStringBinary(this.row));
186     int colCount = 0;
187     // iterate through all column families affected
188     for (Map.Entry<byte [], List<Cell>> entry : this.familyMap.entrySet()) {
189       // map from this family to details for each cell affected within the family
190       List<Map<String, Object>> qualifierDetails = new ArrayList<Map<String, Object>>();
191       columns.put(Bytes.toStringBinary(entry.getKey()), qualifierDetails);
192       colCount += entry.getValue().size();
193       if (maxCols <= 0) {
194         continue;
195       }
196       // add details for each cell
197       for (Cell cell: entry.getValue()) {
198         if (--maxCols <= 0) {
199           continue;
200         }
201         Map<String, Object> cellMap = cellToStringMap(cell);
202         qualifierDetails.add(cellMap);
203       }
204     }
205     map.put("totalColumns", colCount);
206     // add the id if set
207     if (getId() != null) {
208       map.put("id", getId());
209     }
210     // Add the TTL if set
211     // Long.MAX_VALUE is the default, and is interpreted to mean this attribute
212     // has not been set.
213     if (getTTL() != Long.MAX_VALUE) {
214       map.put("ttl", getTTL());
215     }
216     return map;
217   }
218 
219   private static Map<String, Object> cellToStringMap(Cell c) {
220     Map<String, Object> stringMap = new HashMap<String, Object>();
221     stringMap.put("qualifier", Bytes.toStringBinary(c.getQualifierArray(), c.getQualifierOffset(),
222                 c.getQualifierLength()));
223     stringMap.put("timestamp", c.getTimestamp());
224     stringMap.put("vlen", c.getValueLength());
225     List<Tag> tags = CellUtil.getTags(c);
226     if (tags != null) {
227       List<String> tagsString = new ArrayList<String>();
228       for (Tag t : tags) {
229         tagsString.add((t.getType()) + ":" + Bytes.toStringBinary(TagUtil.cloneValue(t)));
230       }
231       stringMap.put("tag", tagsString);
232     }
233     return stringMap;
234   }
235 
236   /**
237    * Set the durability for this mutation
238    * @param d
239    */
240   public Mutation setDurability(Durability d) {
241     this.durability = d;
242     return this;
243   }
244 
245   /** Get the current durability */
246   public Durability getDurability() {
247     return this.durability;
248   }
249 
250   /**
251    * Method for retrieving the put's familyMap
252    * @return familyMap
253    */
254   public NavigableMap<byte [], List<Cell>> getFamilyCellMap() {
255     return this.familyMap;
256   }
257 
258   /**
259    * Method for setting the put's familyMap
260    */
261   public Mutation setFamilyCellMap(NavigableMap<byte [], List<Cell>> map) {
262     // TODO: Shut this down or move it up to be a Constructor.  Get new object rather than change
263     // this internal data member.
264     this.familyMap = map;
265     return this;
266   }
267 
268   /**
269    * Method to check if the familyMap is empty
270    * @return true if empty, false otherwise
271    */
272   public boolean isEmpty() {
273     return familyMap.isEmpty();
274   }
275 
276   /**
277    * Method for retrieving the delete's row
278    * @return row
279    */
280   @Override
281   public byte [] getRow() {
282     return this.row;
283   }
284 
285   @Override
286   public int compareTo(final Row d) {
287     return Bytes.compareTo(this.getRow(), d.getRow());
288   }
289 
290   /**
291    * Method for retrieving the timestamp
292    * @return timestamp
293    */
294   public long getTimeStamp() {
295     return this.ts;
296   }
297 
298   /**
299    * Marks that the clusters with the given clusterIds have consumed the mutation
300    * @param clusterIds of the clusters that have consumed the mutation
301    */
302   public Mutation setClusterIds(List<UUID> clusterIds) {
303     ByteArrayDataOutput out = ByteStreams.newDataOutput();
304     out.writeInt(clusterIds.size());
305     for (UUID clusterId : clusterIds) {
306       out.writeLong(clusterId.getMostSignificantBits());
307       out.writeLong(clusterId.getLeastSignificantBits());
308     }
309     setAttribute(CONSUMED_CLUSTER_IDS, out.toByteArray());
310     return this;
311   }
312 
313   /**
314    * @return the set of clusterIds that have consumed the mutation
315    */
316   public List<UUID> getClusterIds() {
317     List<UUID> clusterIds = new ArrayList<UUID>();
318     byte[] bytes = getAttribute(CONSUMED_CLUSTER_IDS);
319     if(bytes != null) {
320       ByteArrayDataInput in = ByteStreams.newDataInput(bytes);
321       int numClusters = in.readInt();
322       for(int i=0; i<numClusters; i++){
323         clusterIds.add(new UUID(in.readLong(), in.readLong()));
324       }
325     }
326     return clusterIds;
327   }
328 
329   /**
330    * Sets the visibility expression associated with cells in this Mutation.
331    * It is illegal to set <code>CellVisibility</code> on <code>Delete</code> mutation.
332    * @param expression
333    */
334   public Mutation setCellVisibility(CellVisibility expression) {
335     this.setAttribute(VisibilityConstants.VISIBILITY_LABELS_ATTR_KEY, ProtobufUtil
336         .toCellVisibility(expression).toByteArray());
337     return this;
338   }
339 
340   /**
341    * @return CellVisibility associated with cells in this Mutation.
342    * @throws DeserializationException
343    */
344   public CellVisibility getCellVisibility() throws DeserializationException {
345     byte[] cellVisibilityBytes = this.getAttribute(VisibilityConstants.VISIBILITY_LABELS_ATTR_KEY);
346     if (cellVisibilityBytes == null) return null;
347     return ProtobufUtil.toCellVisibility(cellVisibilityBytes);
348   }
349 
350   /**
351    * Number of KeyValues carried by this Mutation.
352    * @return the total number of KeyValues
353    */
354   public int size() {
355     int size = 0;
356     for (List<Cell> cells : this.familyMap.values()) {
357       size += cells.size();
358     }
359     return size;
360   }
361 
362   /**
363    * @return the number of different families
364    */
365   public int numFamilies() {
366     return familyMap.size();
367   }
368 
369   /**
370    * @return Calculate what Mutation adds to class heap size.
371    */
372   @Override
373   public long heapSize() {
374     long heapsize = MUTATION_OVERHEAD;
375     // Adding row
376     heapsize += ClassSize.align(ClassSize.ARRAY + this.row.length);
377 
378     // Adding map overhead
379     heapsize +=
380       ClassSize.align(this.familyMap.size() * ClassSize.MAP_ENTRY);
381     for(Map.Entry<byte [], List<Cell>> entry : this.familyMap.entrySet()) {
382       //Adding key overhead
383       heapsize +=
384         ClassSize.align(ClassSize.ARRAY + entry.getKey().length);
385 
386       //This part is kinds tricky since the JVM can reuse references if you
387       //store the same value, but have a good match with SizeOf at the moment
388       //Adding value overhead
389       heapsize += ClassSize.align(ClassSize.ARRAYLIST);
390       int size = entry.getValue().size();
391       heapsize += ClassSize.align(ClassSize.ARRAY +
392           size * ClassSize.REFERENCE);
393 
394       for(Cell cell : entry.getValue()) {
395         heapsize += CellUtil.estimatedHeapSizeOf(cell);
396       }
397     }
398     heapsize += getAttributeSize();
399     heapsize += extraHeapSize();
400     return ClassSize.align(heapsize);
401   }
402 
403   /**
404    * @return The serialized ACL for this operation, or null if none
405    */
406   public byte[] getACL() {
407     return getAttribute(AccessControlConstants.OP_ATTRIBUTE_ACL);
408   }
409 
410   /**
411    * @param user User short name
412    * @param perms Permissions for the user
413    */
414   public Mutation setACL(String user, Permission perms) {
415     setAttribute(AccessControlConstants.OP_ATTRIBUTE_ACL,
416       ProtobufUtil.toUsersAndPermissions(user, perms).toByteArray());
417     return this;
418   }
419 
420   /**
421    * @param perms A map of permissions for a user or users
422    */
423   public Mutation setACL(Map<String, Permission> perms) {
424     ListMultimap<String, Permission> permMap = ArrayListMultimap.create();
425     for (Map.Entry<String, Permission> entry : perms.entrySet()) {
426       permMap.put(entry.getKey(), entry.getValue());
427     }
428     setAttribute(AccessControlConstants.OP_ATTRIBUTE_ACL,
429       ProtobufUtil.toUsersAndPermissions(permMap).toByteArray());
430     return this;
431   }
432 
433   /**
434    * Return the TTL requested for the result of the mutation, in milliseconds.
435    * @return the TTL requested for the result of the mutation, in milliseconds,
436    * or Long.MAX_VALUE if unset
437    */
438   public long getTTL() {
439     byte[] ttlBytes = getAttribute(OP_ATTRIBUTE_TTL);
440     if (ttlBytes != null) {
441       return Bytes.toLong(ttlBytes);
442     }
443     return Long.MAX_VALUE;
444   }
445 
446   /**
447    * Set the TTL desired for the result of the mutation, in milliseconds.
448    * @param ttl the TTL desired for the result of the mutation, in milliseconds
449    * @return this
450    */
451   public Mutation setTTL(long ttl) {
452     setAttribute(OP_ATTRIBUTE_TTL, Bytes.toBytes(ttl));
453     return this;
454   }
455 
456   /**
457    * @return current value for returnResults
458    */
459   // Used by Increment and Append only.
460   @InterfaceAudience.Private
461   protected boolean isReturnResults() {
462     byte[] v = getAttribute(RETURN_RESULTS);
463     return v == null ? true : Bytes.toBoolean(v);
464   }
465 
466   @InterfaceAudience.Private
467   // Used by Increment and Append only.
468   protected Mutation setReturnResults(boolean returnResults) {
469     setAttribute(RETURN_RESULTS, Bytes.toBytes(returnResults));
470     return this;
471   }
472 
473   /**
474    * Subclasses should override this method to add the heap size of their own fields.
475    * @return the heap size to add (will be aligned).
476    */
477   protected long extraHeapSize(){
478     return 0L;
479   }
480 
481 
482   /**
483    * @param row Row to check
484    * @throws IllegalArgumentException Thrown if <code>row</code> is empty or null or
485    * &gt; {@link HConstants#MAX_ROW_LENGTH}
486    * @return <code>row</code>
487    */
488   static byte [] checkRow(final byte [] row) {
489     return checkRow(row, 0, row == null? 0: row.length);
490   }
491 
492   /**
493    * @param row Row to check
494    * @param offset
495    * @param length
496    * @throws IllegalArgumentException Thrown if <code>row</code> is empty or null or
497    * &gt; {@link HConstants#MAX_ROW_LENGTH}
498    * @return <code>row</code>
499    */
500   static byte [] checkRow(final byte [] row, final int offset, final int length) {
501     if (row == null) {
502       throw new IllegalArgumentException("Row buffer is null");
503     }
504     if (length == 0) {
505       throw new IllegalArgumentException("Row length is 0");
506     }
507     if (length > HConstants.MAX_ROW_LENGTH) {
508       throw new IllegalArgumentException("Row length " + length + " is > " +
509         HConstants.MAX_ROW_LENGTH);
510     }
511     return row;
512   }
513 
514   static void checkRow(ByteBuffer row) {
515     if (row == null) {
516       throw new IllegalArgumentException("Row buffer is null");
517     }
518     if (row.remaining() == 0) {
519       throw new IllegalArgumentException("Row length is 0");
520     }
521     if (row.remaining() > HConstants.MAX_ROW_LENGTH) {
522       throw new IllegalArgumentException("Row length " + row.remaining() + " is > " +
523           HConstants.MAX_ROW_LENGTH);
524     }
525   }
526 }