View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.client;
20  
21  import java.nio.ByteBuffer;
22  import java.util.ArrayList;
23  import java.util.Arrays;
24  import java.util.HashMap;
25  import java.util.List;
26  import java.util.Map;
27  import java.util.NavigableMap;
28  import java.util.TreeMap;
29  import java.util.UUID;
30  
31  import org.apache.hadoop.hbase.Cell;
32  import org.apache.hadoop.hbase.CellScannable;
33  import org.apache.hadoop.hbase.CellScanner;
34  import org.apache.hadoop.hbase.CellUtil;
35  import org.apache.hadoop.hbase.HConstants;
36  import org.apache.hadoop.hbase.KeyValue;
37  import org.apache.hadoop.hbase.Tag;
38  import org.apache.hadoop.hbase.TagUtil;
39  import org.apache.hadoop.hbase.classification.InterfaceAudience;
40  import org.apache.hadoop.hbase.classification.InterfaceStability;
41  import org.apache.hadoop.hbase.exceptions.DeserializationException;
42  import org.apache.hadoop.hbase.io.HeapSize;
43  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
44  import org.apache.hadoop.hbase.security.access.AccessControlConstants;
45  import org.apache.hadoop.hbase.security.access.Permission;
46  import org.apache.hadoop.hbase.security.visibility.CellVisibility;
47  import org.apache.hadoop.hbase.security.visibility.VisibilityConstants;
48  import org.apache.hadoop.hbase.util.Bytes;
49  import org.apache.hadoop.hbase.util.ClassSize;
50  
51  import com.google.common.collect.ArrayListMultimap;
52  import com.google.common.collect.ListMultimap;
53  import com.google.common.io.ByteArrayDataInput;
54  import com.google.common.io.ByteArrayDataOutput;
55  import com.google.common.io.ByteStreams;
56  
57  @InterfaceAudience.Public
58  @InterfaceStability.Evolving
59  public abstract class Mutation extends OperationWithAttributes implements Row, CellScannable,
60      HeapSize {
61    public static final long MUTATION_OVERHEAD = ClassSize.align(
62        // This
63        ClassSize.OBJECT +
64        // row + OperationWithAttributes.attributes
65        2 * ClassSize.REFERENCE +
66        // Timestamp
67        1 * Bytes.SIZEOF_LONG +
68        // durability
69        ClassSize.REFERENCE +
70        // familyMap
71        ClassSize.REFERENCE +
72        // familyMap
73        ClassSize.TREEMAP);
74  
75    /**
76     * The attribute for storing the list of clusters that have consumed the change.
77     */
78    private static final String CONSUMED_CLUSTER_IDS = "_cs.id";
79  
80    /**
81     * The attribute for storing TTL for the result of the mutation.
82     */
83    private static final String OP_ATTRIBUTE_TTL = "_ttl";
84  
85    protected byte [] row = null;
86    protected long ts = HConstants.LATEST_TIMESTAMP;
87    protected Durability durability = Durability.USE_DEFAULT;
88  
89    // A Map sorted by column family.
90    protected NavigableMap<byte [], List<Cell>> familyMap =
91      new TreeMap<byte [], List<Cell>>(Bytes.BYTES_COMPARATOR);
92  
93    @Override
94    public CellScanner cellScanner() {
95      return CellUtil.createCellScanner(getFamilyCellMap());
96    }
97  
98    /**
99     * Creates an empty list if one doesn't exist for the given column family
100    * or else it returns the associated list of Cell objects.
101    *
102    * @param family column family
103    * @return a list of Cell objects, returns an empty list if one doesn't exist.
104    */
105   List<Cell> getCellList(byte[] family) {
106     List<Cell> list = this.familyMap.get(family);
107     if (list == null) {
108       list = new ArrayList<Cell>();
109     }
110     return list;
111   }
112 
113   /*
114    * Create a KeyValue with this objects row key and the Put identifier.
115    *
116    * @return a KeyValue with this objects row key and the Put identifier.
117    */
118   KeyValue createPutKeyValue(byte[] family, byte[] qualifier, long ts, byte[] value) {
119     return new KeyValue(this.row, family, qualifier, ts, KeyValue.Type.Put, value);
120   }
121 
122   /**
123    * Create a KeyValue with this objects row key and the Put identifier.
124    * @param family
125    * @param qualifier
126    * @param ts
127    * @param value
128    * @param tags - Specify the Tags as an Array
129    * @return a KeyValue with this objects row key and the Put identifier.
130    */
131   KeyValue createPutKeyValue(byte[] family, byte[] qualifier, long ts, byte[] value, Tag[] tags) {
132     KeyValue kvWithTag = new KeyValue(this.row, family, qualifier, ts, value, tags);
133     return kvWithTag;
134   }
135 
136   /*
137    * Create a KeyValue with this objects row key and the Put identifier.
138    *
139    * @return a KeyValue with this objects row key and the Put identifier.
140    */
141   KeyValue createPutKeyValue(byte[] family, ByteBuffer qualifier, long ts, ByteBuffer value,
142       Tag[] tags) {
143     return new KeyValue(this.row, 0, this.row == null ? 0 : this.row.length,
144         family, 0, family == null ? 0 : family.length,
145         qualifier, ts, KeyValue.Type.Put, value, tags != null ? Arrays.asList(tags) : null);
146   }
147 
148   /**
149    * Compile the column family (i.e. schema) information
150    * into a Map. Useful for parsing and aggregation by debugging,
151    * logging, and administration tools.
152    * @return Map
153    */
154   @Override
155   public Map<String, Object> getFingerprint() {
156     Map<String, Object> map = new HashMap<String, Object>();
157     List<String> families = new ArrayList<String>();
158     // ideally, we would also include table information, but that information
159     // is not stored in each Operation instance.
160     map.put("families", families);
161     for (Map.Entry<byte [], List<Cell>> entry : this.familyMap.entrySet()) {
162       families.add(Bytes.toStringBinary(entry.getKey()));
163     }
164     return map;
165   }
166 
167   /**
168    * Compile the details beyond the scope of getFingerprint (row, columns,
169    * timestamps, etc.) into a Map along with the fingerprinted information.
170    * Useful for debugging, logging, and administration tools.
171    * @param maxCols a limit on the number of columns output prior to truncation
172    * @return Map
173    */
174   @Override
175   public Map<String, Object> toMap(int maxCols) {
176     // we start with the fingerprint map and build on top of it.
177     Map<String, Object> map = getFingerprint();
178     // replace the fingerprint's simple list of families with a
179     // map from column families to lists of qualifiers and kv details
180     Map<String, List<Map<String, Object>>> columns =
181       new HashMap<String, List<Map<String, Object>>>();
182     map.put("families", columns);
183     map.put("row", Bytes.toStringBinary(this.row));
184     int colCount = 0;
185     // iterate through all column families affected
186     for (Map.Entry<byte [], List<Cell>> entry : this.familyMap.entrySet()) {
187       // map from this family to details for each cell affected within the family
188       List<Map<String, Object>> qualifierDetails = new ArrayList<Map<String, Object>>();
189       columns.put(Bytes.toStringBinary(entry.getKey()), qualifierDetails);
190       colCount += entry.getValue().size();
191       if (maxCols <= 0) {
192         continue;
193       }
194       // add details for each cell
195       for (Cell cell: entry.getValue()) {
196         if (--maxCols <= 0) {
197           continue;
198         }
199         Map<String, Object> cellMap = cellToStringMap(cell);
200         qualifierDetails.add(cellMap);
201       }
202     }
203     map.put("totalColumns", colCount);
204     // add the id if set
205     if (getId() != null) {
206       map.put("id", getId());
207     }
208     // Add the TTL if set
209     // Long.MAX_VALUE is the default, and is interpreted to mean this attribute
210     // has not been set.
211     if (getTTL() != Long.MAX_VALUE) {
212       map.put("ttl", getTTL());
213     }
214     return map;
215   }
216 
217   private static Map<String, Object> cellToStringMap(Cell c) {
218     Map<String, Object> stringMap = new HashMap<String, Object>();
219     stringMap.put("qualifier", Bytes.toStringBinary(c.getQualifierArray(), c.getQualifierOffset(),
220                 c.getQualifierLength()));
221     stringMap.put("timestamp", c.getTimestamp());
222     stringMap.put("vlen", c.getValueLength());
223     List<Tag> tags = CellUtil.getTags(c);
224     if (tags != null) {
225       List<String> tagsString = new ArrayList<String>();
226       for (Tag t : tags) {
227         tagsString.add((t.getType()) + ":" + Bytes.toStringBinary(TagUtil.cloneValue(t)));
228       }
229       stringMap.put("tag", tagsString);
230     }
231     return stringMap;
232   }
233 
234   /**
235    * Set the durability for this mutation
236    * @param d
237    */
238   public Mutation setDurability(Durability d) {
239     this.durability = d;
240     return this;
241   }
242 
243   /** Get the current durability */
244   public Durability getDurability() {
245     return this.durability;
246   }
247 
248   /**
249    * Method for retrieving the put's familyMap
250    * @return familyMap
251    */
252   public NavigableMap<byte [], List<Cell>> getFamilyCellMap() {
253     return this.familyMap;
254   }
255 
256   /**
257    * Method for setting the put's familyMap
258    */
259   public Mutation setFamilyCellMap(NavigableMap<byte [], List<Cell>> map) {
260     // TODO: Shut this down or move it up to be a Constructor.  Get new object rather than change
261     // this internal data member.
262     this.familyMap = map;
263     return this;
264   }
265 
266   /**
267    * Method to check if the familyMap is empty
268    * @return true if empty, false otherwise
269    */
270   public boolean isEmpty() {
271     return familyMap.isEmpty();
272   }
273 
274   /**
275    * Method for retrieving the delete's row
276    * @return row
277    */
278   @Override
279   public byte [] getRow() {
280     return this.row;
281   }
282 
283   @Override
284   public int compareTo(final Row d) {
285     return Bytes.compareTo(this.getRow(), d.getRow());
286   }
287 
288   /**
289    * Method for retrieving the timestamp
290    * @return timestamp
291    */
292   public long getTimeStamp() {
293     return this.ts;
294   }
295 
296   /**
297    * Marks that the clusters with the given clusterIds have consumed the mutation
298    * @param clusterIds of the clusters that have consumed the mutation
299    */
300   public Mutation setClusterIds(List<UUID> clusterIds) {
301     ByteArrayDataOutput out = ByteStreams.newDataOutput();
302     out.writeInt(clusterIds.size());
303     for (UUID clusterId : clusterIds) {
304       out.writeLong(clusterId.getMostSignificantBits());
305       out.writeLong(clusterId.getLeastSignificantBits());
306     }
307     setAttribute(CONSUMED_CLUSTER_IDS, out.toByteArray());
308     return this;
309   }
310 
311   /**
312    * @return the set of clusterIds that have consumed the mutation
313    */
314   public List<UUID> getClusterIds() {
315     List<UUID> clusterIds = new ArrayList<UUID>();
316     byte[] bytes = getAttribute(CONSUMED_CLUSTER_IDS);
317     if(bytes != null) {
318       ByteArrayDataInput in = ByteStreams.newDataInput(bytes);
319       int numClusters = in.readInt();
320       for(int i=0; i<numClusters; i++){
321         clusterIds.add(new UUID(in.readLong(), in.readLong()));
322       }
323     }
324     return clusterIds;
325   }
326 
327   /**
328    * Sets the visibility expression associated with cells in this Mutation.
329    * It is illegal to set <code>CellVisibility</code> on <code>Delete</code> mutation.
330    * @param expression
331    */
332   public Mutation setCellVisibility(CellVisibility expression) {
333     this.setAttribute(VisibilityConstants.VISIBILITY_LABELS_ATTR_KEY, ProtobufUtil
334         .toCellVisibility(expression).toByteArray());
335     return this;
336   }
337 
338   /**
339    * @return CellVisibility associated with cells in this Mutation.
340    * @throws DeserializationException
341    */
342   public CellVisibility getCellVisibility() throws DeserializationException {
343     byte[] cellVisibilityBytes = this.getAttribute(VisibilityConstants.VISIBILITY_LABELS_ATTR_KEY);
344     if (cellVisibilityBytes == null) return null;
345     return ProtobufUtil.toCellVisibility(cellVisibilityBytes);
346   }
347 
348   /**
349    * Number of KeyValues carried by this Mutation.
350    * @return the total number of KeyValues
351    */
352   public int size() {
353     int size = 0;
354     for (List<Cell> cells : this.familyMap.values()) {
355       size += cells.size();
356     }
357     return size;
358   }
359 
360   /**
361    * @return the number of different families
362    */
363   public int numFamilies() {
364     return familyMap.size();
365   }
366 
367   /**
368    * @return Calculate what Mutation adds to class heap size.
369    */
370   @Override
371   public long heapSize() {
372     long heapsize = MUTATION_OVERHEAD;
373     // Adding row
374     heapsize += ClassSize.align(ClassSize.ARRAY + this.row.length);
375 
376     // Adding map overhead
377     heapsize +=
378       ClassSize.align(this.familyMap.size() * ClassSize.MAP_ENTRY);
379     for(Map.Entry<byte [], List<Cell>> entry : this.familyMap.entrySet()) {
380       //Adding key overhead
381       heapsize +=
382         ClassSize.align(ClassSize.ARRAY + entry.getKey().length);
383 
384       //This part is kinds tricky since the JVM can reuse references if you
385       //store the same value, but have a good match with SizeOf at the moment
386       //Adding value overhead
387       heapsize += ClassSize.align(ClassSize.ARRAYLIST);
388       int size = entry.getValue().size();
389       heapsize += ClassSize.align(ClassSize.ARRAY +
390           size * ClassSize.REFERENCE);
391 
392       for(Cell cell : entry.getValue()) {
393         heapsize += CellUtil.estimatedHeapSizeOf(cell);
394       }
395     }
396     heapsize += getAttributeSize();
397     heapsize += extraHeapSize();
398     return ClassSize.align(heapsize);
399   }
400 
401   /**
402    * @return The serialized ACL for this operation, or null if none
403    */
404   public byte[] getACL() {
405     return getAttribute(AccessControlConstants.OP_ATTRIBUTE_ACL);
406   }
407 
408   /**
409    * @param user User short name
410    * @param perms Permissions for the user
411    */
412   public Mutation setACL(String user, Permission perms) {
413     setAttribute(AccessControlConstants.OP_ATTRIBUTE_ACL,
414       ProtobufUtil.toUsersAndPermissions(user, perms).toByteArray());
415     return this;
416   }
417 
418   /**
419    * @param perms A map of permissions for a user or users
420    */
421   public Mutation setACL(Map<String, Permission> perms) {
422     ListMultimap<String, Permission> permMap = ArrayListMultimap.create();
423     for (Map.Entry<String, Permission> entry : perms.entrySet()) {
424       permMap.put(entry.getKey(), entry.getValue());
425     }
426     setAttribute(AccessControlConstants.OP_ATTRIBUTE_ACL,
427       ProtobufUtil.toUsersAndPermissions(permMap).toByteArray());
428     return this;
429   }
430 
431   /**
432    * Return the TTL requested for the result of the mutation, in milliseconds.
433    * @return the TTL requested for the result of the mutation, in milliseconds,
434    * or Long.MAX_VALUE if unset
435    */
436   public long getTTL() {
437     byte[] ttlBytes = getAttribute(OP_ATTRIBUTE_TTL);
438     if (ttlBytes != null) {
439       return Bytes.toLong(ttlBytes);
440     }
441     return Long.MAX_VALUE;
442   }
443 
444   /**
445    * Set the TTL desired for the result of the mutation, in milliseconds.
446    * @param ttl the TTL desired for the result of the mutation, in milliseconds
447    * @return this
448    */
449   public Mutation setTTL(long ttl) {
450     setAttribute(OP_ATTRIBUTE_TTL, Bytes.toBytes(ttl));
451     return this;
452   }
453 
454   /**
455    * Subclasses should override this method to add the heap size of their own fields.
456    * @return the heap size to add (will be aligned).
457    */
458   protected long extraHeapSize(){
459     return 0L;
460   }
461 
462 
463   /**
464    * @param row Row to check
465    * @throws IllegalArgumentException Thrown if <code>row</code> is empty or null or
466    * &gt; {@link HConstants#MAX_ROW_LENGTH}
467    * @return <code>row</code>
468    */
469   static byte [] checkRow(final byte [] row) {
470     return checkRow(row, 0, row == null? 0: row.length);
471   }
472 
473   /**
474    * @param row Row to check
475    * @param offset
476    * @param length
477    * @throws IllegalArgumentException Thrown if <code>row</code> is empty or null or
478    * &gt; {@link HConstants#MAX_ROW_LENGTH}
479    * @return <code>row</code>
480    */
481   static byte [] checkRow(final byte [] row, final int offset, final int length) {
482     if (row == null) {
483       throw new IllegalArgumentException("Row buffer is null");
484     }
485     if (length == 0) {
486       throw new IllegalArgumentException("Row length is 0");
487     }
488     if (length > HConstants.MAX_ROW_LENGTH) {
489       throw new IllegalArgumentException("Row length " + length + " is > " +
490         HConstants.MAX_ROW_LENGTH);
491     }
492     return row;
493   }
494 
495   static void checkRow(ByteBuffer row) {
496     if (row == null) {
497       throw new IllegalArgumentException("Row buffer is null");
498     }
499     if (row.remaining() == 0) {
500       throw new IllegalArgumentException("Row length is 0");
501     }
502     if (row.remaining() > HConstants.MAX_ROW_LENGTH) {
503       throw new IllegalArgumentException("Row length " + row.remaining() + " is > " +
504           HConstants.MAX_ROW_LENGTH);
505     }
506   }
507 }