View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.client;
20  
21  import java.nio.ByteBuffer;
22  import java.util.ArrayList;
23  import java.util.Arrays;
24  import java.util.HashMap;
25  import java.util.List;
26  import java.util.Map;
27  import java.util.NavigableMap;
28  import java.util.TreeMap;
29  import java.util.UUID;
30  
31  import org.apache.hadoop.hbase.Cell;
32  import org.apache.hadoop.hbase.CellScannable;
33  import org.apache.hadoop.hbase.CellScanner;
34  import org.apache.hadoop.hbase.CellUtil;
35  import org.apache.hadoop.hbase.HConstants;
36  import org.apache.hadoop.hbase.KeyValue;
37  import org.apache.hadoop.hbase.Tag;
38  import org.apache.hadoop.hbase.classification.InterfaceAudience;
39  import org.apache.hadoop.hbase.classification.InterfaceStability;
40  import org.apache.hadoop.hbase.exceptions.DeserializationException;
41  import org.apache.hadoop.hbase.io.HeapSize;
42  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
43  import org.apache.hadoop.hbase.security.access.AccessControlConstants;
44  import org.apache.hadoop.hbase.security.access.Permission;
45  import org.apache.hadoop.hbase.security.visibility.CellVisibility;
46  import org.apache.hadoop.hbase.security.visibility.VisibilityConstants;
47  import org.apache.hadoop.hbase.util.Bytes;
48  import org.apache.hadoop.hbase.util.ClassSize;
49  
50  import com.google.common.collect.ArrayListMultimap;
51  import com.google.common.collect.ListMultimap;
52  import com.google.common.io.ByteArrayDataInput;
53  import com.google.common.io.ByteArrayDataOutput;
54  import com.google.common.io.ByteStreams;
55  
56  @InterfaceAudience.Public
57  @InterfaceStability.Evolving
58  public abstract class Mutation extends OperationWithAttributes implements Row, CellScannable,
59      HeapSize {
60    public static final long MUTATION_OVERHEAD = ClassSize.align(
61        // This
62        ClassSize.OBJECT +
63        // row + OperationWithAttributes.attributes
64        2 * ClassSize.REFERENCE +
65        // Timestamp
66        1 * Bytes.SIZEOF_LONG +
67        // durability
68        ClassSize.REFERENCE +
69        // familyMap
70        ClassSize.REFERENCE +
71        // familyMap
72        ClassSize.TREEMAP);
73  
74    /**
75     * The attribute for storing the list of clusters that have consumed the change.
76     */
77    private static final String CONSUMED_CLUSTER_IDS = "_cs.id";
78  
79    /**
80     * The attribute for storing TTL for the result of the mutation.
81     */
82    private static final String OP_ATTRIBUTE_TTL = "_ttl";
83  
84    protected byte [] row = null;
85    protected long ts = HConstants.LATEST_TIMESTAMP;
86    protected Durability durability = Durability.USE_DEFAULT;
87  
88    // A Map sorted by column family.
89    protected NavigableMap<byte [], List<Cell>> familyMap =
90      new TreeMap<byte [], List<Cell>>(Bytes.BYTES_COMPARATOR);
91  
92    @Override
93    public CellScanner cellScanner() {
94      return CellUtil.createCellScanner(getFamilyCellMap());
95    }
96  
97    /**
98     * Creates an empty list if one doesn't exist for the given column family
99     * or else it returns the associated list of Cell objects.
100    *
101    * @param family column family
102    * @return a list of Cell objects, returns an empty list if one doesn't exist.
103    */
104   List<Cell> getCellList(byte[] family) {
105     List<Cell> list = this.familyMap.get(family);
106     if (list == null) {
107       list = new ArrayList<Cell>();
108     }
109     return list;
110   }
111 
112   /*
113    * Create a KeyValue with this objects row key and the Put identifier.
114    *
115    * @return a KeyValue with this objects row key and the Put identifier.
116    */
117   KeyValue createPutKeyValue(byte[] family, byte[] qualifier, long ts, byte[] value) {
118     return new KeyValue(this.row, family, qualifier, ts, KeyValue.Type.Put, value);
119   }
120 
121   /**
122    * Create a KeyValue with this objects row key and the Put identifier.
123    * @param family
124    * @param qualifier
125    * @param ts
126    * @param value
127    * @param tags - Specify the Tags as an Array {@link KeyValue.Tag}
128    * @return a KeyValue with this objects row key and the Put identifier.
129    */
130   KeyValue createPutKeyValue(byte[] family, byte[] qualifier, long ts, byte[] value, Tag[] tags) {
131     KeyValue kvWithTag = new KeyValue(this.row, family, qualifier, ts, value, tags);
132     return kvWithTag;
133   }
134 
135   /*
136    * Create a KeyValue with this objects row key and the Put identifier.
137    *
138    * @return a KeyValue with this objects row key and the Put identifier.
139    */
140   KeyValue createPutKeyValue(byte[] family, ByteBuffer qualifier, long ts, ByteBuffer value,
141                              Tag[] tags) {
142     return new KeyValue(this.row, 0, this.row == null ? 0 : this.row.length,
143         family, 0, family == null ? 0 : family.length,
144         qualifier, ts, KeyValue.Type.Put, value, tags != null ? Arrays.asList(tags) : null);
145   }
146 
147   /**
148    * Compile the column family (i.e. schema) information
149    * into a Map. Useful for parsing and aggregation by debugging,
150    * logging, and administration tools.
151    * @return Map
152    */
153   @Override
154   public Map<String, Object> getFingerprint() {
155     Map<String, Object> map = new HashMap<String, Object>();
156     List<String> families = new ArrayList<String>();
157     // ideally, we would also include table information, but that information
158     // is not stored in each Operation instance.
159     map.put("families", families);
160     for (Map.Entry<byte [], List<Cell>> entry : this.familyMap.entrySet()) {
161       families.add(Bytes.toStringBinary(entry.getKey()));
162     }
163     return map;
164   }
165 
166   /**
167    * Compile the details beyond the scope of getFingerprint (row, columns,
168    * timestamps, etc.) into a Map along with the fingerprinted information.
169    * Useful for debugging, logging, and administration tools.
170    * @param maxCols a limit on the number of columns output prior to truncation
171    * @return Map
172    */
173   @Override
174   public Map<String, Object> toMap(int maxCols) {
175     // we start with the fingerprint map and build on top of it.
176     Map<String, Object> map = getFingerprint();
177     // replace the fingerprint's simple list of families with a
178     // map from column families to lists of qualifiers and kv details
179     Map<String, List<Map<String, Object>>> columns =
180       new HashMap<String, List<Map<String, Object>>>();
181     map.put("families", columns);
182     map.put("row", Bytes.toStringBinary(this.row));
183     int colCount = 0;
184     // iterate through all column families affected
185     for (Map.Entry<byte [], List<Cell>> entry : this.familyMap.entrySet()) {
186       // map from this family to details for each cell affected within the family
187       List<Map<String, Object>> qualifierDetails = new ArrayList<Map<String, Object>>();
188       columns.put(Bytes.toStringBinary(entry.getKey()), qualifierDetails);
189       colCount += entry.getValue().size();
190       if (maxCols <= 0) {
191         continue;
192       }
193       // add details for each cell
194       for (Cell cell: entry.getValue()) {
195         if (--maxCols <= 0 ) {
196           continue;
197         }
198         Map<String, Object> cellMap = cellToStringMap(cell);
199         qualifierDetails.add(cellMap);
200       }
201     }
202     map.put("totalColumns", colCount);
203     // add the id if set
204     if (getId() != null) {
205       map.put("id", getId());
206     }
207     // Add the TTL if set
208     // Long.MAX_VALUE is the default, and is interpreted to mean this attribute
209     // has not been set.
210     if (getTTL() != Long.MAX_VALUE) {
211       map.put("ttl", getTTL());
212     }
213     return map;
214   }
215 
216   private static Map<String, Object> cellToStringMap(Cell c) {
217     Map<String, Object> stringMap = new HashMap<String, Object>();
218     stringMap.put("qualifier", Bytes.toStringBinary(c.getQualifierArray(), c.getQualifierOffset(),
219                 c.getQualifierLength()));
220     stringMap.put("timestamp", c.getTimestamp());
221     stringMap.put("vlen", c.getValueLength());
222     List<Tag> tags = Tag.asList(c.getTagsArray(), c.getTagsOffset(), c.getTagsLength());
223     if (tags != null) {
224       List<String> tagsString = new ArrayList<String>();
225       for (Tag t : tags) {
226         tagsString.add((t.getType()) + ":" + Bytes.toStringBinary(t.getValue()));
227       }
228       stringMap.put("tag", tagsString);
229     }
230     return stringMap;
231   }
232 
233   /**
234    * Set the durability for this mutation
235    * @param d
236    */
237   public Mutation setDurability(Durability d) {
238     this.durability = d;
239     return this;
240   }
241 
242   /** Get the current durability */
243   public Durability getDurability() {
244     return this.durability;
245   }
246 
247   /**
248    * Method for retrieving the put's familyMap
249    * @return familyMap
250    */
251   public NavigableMap<byte [], List<Cell>> getFamilyCellMap() {
252     return this.familyMap;
253   }
254 
255   /**
256    * Method for setting the put's familyMap
257    */
258   public Mutation setFamilyCellMap(NavigableMap<byte [], List<Cell>> map) {
259     // TODO: Shut this down or move it up to be a Constructor.  Get new object rather than change
260     // this internal data member.
261     this.familyMap = map;
262     return this;
263   }
264 
265   /**
266    * Method to check if the familyMap is empty
267    * @return true if empty, false otherwise
268    */
269   public boolean isEmpty() {
270     return familyMap.isEmpty();
271   }
272 
273   /**
274    * Method for retrieving the delete's row
275    * @return row
276    */
277   @Override
278   public byte [] getRow() {
279     return this.row;
280   }
281 
282   @Override
283   public int compareTo(final Row d) {
284     return Bytes.compareTo(this.getRow(), d.getRow());
285   }
286 
287   /**
288    * Method for retrieving the timestamp
289    * @return timestamp
290    */
291   public long getTimeStamp() {
292     return this.ts;
293   }
294 
295   /**
296    * Marks that the clusters with the given clusterIds have consumed the mutation
297    * @param clusterIds of the clusters that have consumed the mutation
298    */
299   public Mutation setClusterIds(List<UUID> clusterIds) {
300     ByteArrayDataOutput out = ByteStreams.newDataOutput();
301     out.writeInt(clusterIds.size());
302     for (UUID clusterId : clusterIds) {
303       out.writeLong(clusterId.getMostSignificantBits());
304       out.writeLong(clusterId.getLeastSignificantBits());
305     }
306     setAttribute(CONSUMED_CLUSTER_IDS, out.toByteArray());
307     return this;
308   }
309 
310   /**
311    * @return the set of clusterIds that have consumed the mutation
312    */
313   public List<UUID> getClusterIds() {
314     List<UUID> clusterIds = new ArrayList<UUID>();
315     byte[] bytes = getAttribute(CONSUMED_CLUSTER_IDS);
316     if(bytes != null) {
317       ByteArrayDataInput in = ByteStreams.newDataInput(bytes);
318       int numClusters = in.readInt();
319       for(int i=0; i<numClusters; i++){
320         clusterIds.add(new UUID(in.readLong(), in.readLong()));
321       }
322     }
323     return clusterIds;
324   }
325 
326   /**
327    * Sets the visibility expression associated with cells in this Mutation.
328    * It is illegal to set <code>CellVisibility</code> on <code>Delete</code> mutation.
329    * @param expression
330    */
331   public Mutation setCellVisibility(CellVisibility expression) {
332     this.setAttribute(VisibilityConstants.VISIBILITY_LABELS_ATTR_KEY, ProtobufUtil
333         .toCellVisibility(expression).toByteArray());
334     return this;
335   }
336 
337   /**
338    * @return CellVisibility associated with cells in this Mutation.
339    * @throws DeserializationException
340    */
341   public CellVisibility getCellVisibility() throws DeserializationException {
342     byte[] cellVisibilityBytes = this.getAttribute(VisibilityConstants.VISIBILITY_LABELS_ATTR_KEY);
343     if (cellVisibilityBytes == null) return null;
344     return ProtobufUtil.toCellVisibility(cellVisibilityBytes);
345   }
346 
347   /**
348    * Number of KeyValues carried by this Mutation.
349    * @return the total number of KeyValues
350    */
351   public int size() {
352     int size = 0;
353     for (List<Cell> cells : this.familyMap.values()) {
354       size += cells.size();
355     }
356     return size;
357   }
358 
359   /**
360    * @return the number of different families
361    */
362   public int numFamilies() {
363     return familyMap.size();
364   }
365 
366   /**
367    * @return Calculate what Mutation adds to class heap size.
368    */
369   @Override
370   public long heapSize() {
371     long heapsize = MUTATION_OVERHEAD;
372     // Adding row
373     heapsize += ClassSize.align(ClassSize.ARRAY + this.row.length);
374 
375     // Adding map overhead
376     heapsize +=
377       ClassSize.align(this.familyMap.size() * ClassSize.MAP_ENTRY);
378     for(Map.Entry<byte [], List<Cell>> entry : this.familyMap.entrySet()) {
379       //Adding key overhead
380       heapsize +=
381         ClassSize.align(ClassSize.ARRAY + entry.getKey().length);
382 
383       //This part is kinds tricky since the JVM can reuse references if you
384       //store the same value, but have a good match with SizeOf at the moment
385       //Adding value overhead
386       heapsize += ClassSize.align(ClassSize.ARRAYLIST);
387       int size = entry.getValue().size();
388       heapsize += ClassSize.align(ClassSize.ARRAY +
389           size * ClassSize.REFERENCE);
390 
391       for(Cell cell : entry.getValue()) {
392         heapsize += CellUtil.estimatedHeapSizeOf(cell);
393       }
394     }
395     heapsize += getAttributeSize();
396     heapsize += extraHeapSize();
397     return ClassSize.align(heapsize);
398   }
399 
400   /**
401    * @return The serialized ACL for this operation, or null if none
402    */
403   public byte[] getACL() {
404     return getAttribute(AccessControlConstants.OP_ATTRIBUTE_ACL);
405   }
406 
407   /**
408    * @param user User short name
409    * @param perms Permissions for the user
410    */
411   public Mutation setACL(String user, Permission perms) {
412     setAttribute(AccessControlConstants.OP_ATTRIBUTE_ACL,
413       ProtobufUtil.toUsersAndPermissions(user, perms).toByteArray());
414     return this;
415   }
416 
417   /**
418    * @param perms A map of permissions for a user or users
419    */
420   public Mutation setACL(Map<String, Permission> perms) {
421     ListMultimap<String, Permission> permMap = ArrayListMultimap.create();
422     for (Map.Entry<String, Permission> entry : perms.entrySet()) {
423       permMap.put(entry.getKey(), entry.getValue());
424     }
425     setAttribute(AccessControlConstants.OP_ATTRIBUTE_ACL,
426       ProtobufUtil.toUsersAndPermissions(permMap).toByteArray());
427     return this;
428   }
429 
430   /**
431    * Return the TTL requested for the result of the mutation, in milliseconds.
432    * @return the TTL requested for the result of the mutation, in milliseconds,
433    * or Long.MAX_VALUE if unset
434    */
435   public long getTTL() {
436     byte[] ttlBytes = getAttribute(OP_ATTRIBUTE_TTL);
437     if (ttlBytes != null) {
438       return Bytes.toLong(ttlBytes);
439     }
440     return Long.MAX_VALUE;
441   }
442 
443   /**
444    * Set the TTL desired for the result of the mutation, in milliseconds.
445    * @param ttl the TTL desired for the result of the mutation, in milliseconds
446    * @return this
447    */
448   public Mutation setTTL(long ttl) {
449     setAttribute(OP_ATTRIBUTE_TTL, Bytes.toBytes(ttl));
450     return this;
451   }
452 
453   /**
454    * Subclasses should override this method to add the heap size of their own fields.
455    * @return the heap size to add (will be aligned).
456    */
457   protected long extraHeapSize(){
458     return 0L;
459   }
460 
461 
462   /**
463    * @param row Row to check
464    * @throws IllegalArgumentException Thrown if <code>row</code> is empty or null or
465    * &gt; {@link HConstants#MAX_ROW_LENGTH}
466    * @return <code>row</code>
467    */
468   static byte [] checkRow(final byte [] row) {
469     return checkRow(row, 0, row == null? 0: row.length);
470   }
471 
472   /**
473    * @param row Row to check
474    * @param offset
475    * @param length
476    * @throws IllegalArgumentException Thrown if <code>row</code> is empty or null or
477    * &gt; {@link HConstants#MAX_ROW_LENGTH}
478    * @return <code>row</code>
479    */
480   static byte [] checkRow(final byte [] row, final int offset, final int length) {
481     if (row == null) {
482       throw new IllegalArgumentException("Row buffer is null");
483     }
484     if (length == 0) {
485       throw new IllegalArgumentException("Row length is 0");
486     }
487     if (length > HConstants.MAX_ROW_LENGTH) {
488       throw new IllegalArgumentException("Row length " + length + " is > " +
489         HConstants.MAX_ROW_LENGTH);
490     }
491     return row;
492   }
493 
494   static void checkRow(ByteBuffer row) {
495     if (row == null) {
496       throw new IllegalArgumentException("Row buffer is null");
497     }
498     if (row.remaining() == 0) {
499       throw new IllegalArgumentException("Row length is 0");
500     }
501     if (row.remaining() > HConstants.MAX_ROW_LENGTH) {
502       throw new IllegalArgumentException("Row length " + row.remaining() + " is > " +
503           HConstants.MAX_ROW_LENGTH);
504     }
505   }
506 }