View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.client;
20  
21  import java.nio.ByteBuffer;
22  import java.util.ArrayList;
23  import java.util.Arrays;
24  import java.util.HashMap;
25  import java.util.List;
26  import java.util.Map;
27  import java.util.NavigableMap;
28  import java.util.TreeMap;
29  import java.util.UUID;
30  
31  import org.apache.hadoop.hbase.classification.InterfaceAudience;
32  import org.apache.hadoop.hbase.classification.InterfaceStability;
33  import org.apache.hadoop.hbase.Cell;
34  import org.apache.hadoop.hbase.CellScannable;
35  import org.apache.hadoop.hbase.CellScanner;
36  import org.apache.hadoop.hbase.CellUtil;
37  import org.apache.hadoop.hbase.HConstants;
38  import org.apache.hadoop.hbase.KeyValue;
39  import org.apache.hadoop.hbase.KeyValueUtil;
40  import org.apache.hadoop.hbase.Tag;
41  import org.apache.hadoop.hbase.exceptions.DeserializationException;
42  import org.apache.hadoop.hbase.io.HeapSize;
43  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
44  import org.apache.hadoop.hbase.security.access.AccessControlConstants;
45  import org.apache.hadoop.hbase.security.access.Permission;
46  import org.apache.hadoop.hbase.security.visibility.CellVisibility;
47  import org.apache.hadoop.hbase.security.visibility.VisibilityConstants;
48  import org.apache.hadoop.hbase.util.Bytes;
49  import org.apache.hadoop.hbase.util.ClassSize;
50  
51  import com.google.common.collect.ArrayListMultimap;
52  import com.google.common.collect.ListMultimap;
53  import com.google.common.collect.Lists;
54  import com.google.common.io.ByteArrayDataInput;
55  import com.google.common.io.ByteArrayDataOutput;
56  import com.google.common.io.ByteStreams;
57  
58  @InterfaceAudience.Public
59  @InterfaceStability.Evolving
60  public abstract class Mutation extends OperationWithAttributes implements Row, CellScannable,
61      HeapSize {
62    public static final long MUTATION_OVERHEAD = ClassSize.align(
63        // This
64        ClassSize.OBJECT +
65        // row + OperationWithAttributes.attributes
66        2 * ClassSize.REFERENCE +
67        // Timestamp
68        1 * Bytes.SIZEOF_LONG +
69        // durability
70        ClassSize.REFERENCE +
71        // familyMap
72        ClassSize.REFERENCE +
73        // familyMap
74        ClassSize.TREEMAP);
75  
76    /**
77     * The attribute for storing the list of clusters that have consumed the change.
78     */
79    private static final String CONSUMED_CLUSTER_IDS = "_cs.id";
80  
81    /**
82     * The attribute for storing TTL for the result of the mutation.
83     */
84    private static final String OP_ATTRIBUTE_TTL = "_ttl";
85  
86    protected static final String RETURN_RESULTS = "_rr_";
87  
88    protected byte [] row = null;
89    protected long ts = HConstants.LATEST_TIMESTAMP;
90    protected Durability durability = Durability.USE_DEFAULT;
91  
92    // A Map sorted by column family.
93    protected NavigableMap<byte [], List<Cell>> familyMap =
94      new TreeMap<byte [], List<Cell>>(Bytes.BYTES_COMPARATOR);
95  
96    @Override
97    public CellScanner cellScanner() {
98      return CellUtil.createCellScanner(getFamilyCellMap());
99    }
100 
101   /**
102    * Creates an empty list if one doesn't exist for the given column family
103    * or else it returns the associated list of Cell objects.
104    *
105    * @param family column family
106    * @return a list of Cell objects, returns an empty list if one doesn't exist.
107    */
108   List<Cell> getCellList(byte[] family) {
109     List<Cell> list = this.familyMap.get(family);
110     if (list == null) {
111       list = new ArrayList<Cell>();
112     }
113     return list;
114   }
115 
116   /*
117    * Create a KeyValue with this objects row key and the Put identifier.
118    *
119    * @return a KeyValue with this objects row key and the Put identifier.
120    */
121   KeyValue createPutKeyValue(byte[] family, byte[] qualifier, long ts, byte[] value) {
122     return new KeyValue(this.row, family, qualifier, ts, KeyValue.Type.Put, value);
123   }
124 
125   /**
126    * Create a KeyValue with this objects row key and the Put identifier.
127    * @param family
128    * @param qualifier
129    * @param ts
130    * @param value
131    * @param tags - Specify the Tags as an Array {@link KeyValue.Tag}
132    * @return a KeyValue with this objects row key and the Put identifier.
133    */
134   KeyValue createPutKeyValue(byte[] family, byte[] qualifier, long ts, byte[] value, Tag[] tags) {
135     KeyValue kvWithTag = new KeyValue(this.row, family, qualifier, ts, value, tags);
136     return kvWithTag;
137   }
138 
139   /*
140    * Create a KeyValue with this objects row key and the Put identifier.
141    *
142    * @return a KeyValue with this objects row key and the Put identifier.
143    */
144   KeyValue createPutKeyValue(byte[] family, ByteBuffer qualifier, long ts, ByteBuffer value,
145                              Tag[] tags) {
146     return new KeyValue(this.row, 0, this.row == null ? 0 : this.row.length,
147         family, 0, family == null ? 0 : family.length,
148         qualifier, ts, KeyValue.Type.Put, value, tags != null ? Arrays.asList(tags) : null);
149   }
150 
151   /**
152    * Compile the column family (i.e. schema) information
153    * into a Map. Useful for parsing and aggregation by debugging,
154    * logging, and administration tools.
155    * @return Map
156    */
157   @Override
158   public Map<String, Object> getFingerprint() {
159     Map<String, Object> map = new HashMap<String, Object>();
160     List<String> families = new ArrayList<String>();
161     // ideally, we would also include table information, but that information
162     // is not stored in each Operation instance.
163     map.put("families", families);
164     for (Map.Entry<byte [], List<Cell>> entry : this.familyMap.entrySet()) {
165       families.add(Bytes.toStringBinary(entry.getKey()));
166     }
167     return map;
168   }
169 
170   /**
171    * Compile the details beyond the scope of getFingerprint (row, columns,
172    * timestamps, etc.) into a Map along with the fingerprinted information.
173    * Useful for debugging, logging, and administration tools.
174    * @param maxCols a limit on the number of columns output prior to truncation
175    * @return Map
176    */
177   @Override
178   public Map<String, Object> toMap(int maxCols) {
179     // we start with the fingerprint map and build on top of it.
180     Map<String, Object> map = getFingerprint();
181     // replace the fingerprint's simple list of families with a
182     // map from column families to lists of qualifiers and kv details
183     Map<String, List<Map<String, Object>>> columns =
184       new HashMap<String, List<Map<String, Object>>>();
185     map.put("families", columns);
186     map.put("row", Bytes.toStringBinary(this.row));
187     int colCount = 0;
188     // iterate through all column families affected
189     for (Map.Entry<byte [], List<Cell>> entry : this.familyMap.entrySet()) {
190       // map from this family to details for each cell affected within the family
191       List<Map<String, Object>> qualifierDetails = new ArrayList<Map<String, Object>>();
192       columns.put(Bytes.toStringBinary(entry.getKey()), qualifierDetails);
193       colCount += entry.getValue().size();
194       if (maxCols <= 0) {
195         continue;
196       }
197       // add details for each cell
198       for (Cell cell: entry.getValue()) {
199         if (--maxCols <= 0 ) {
200           continue;
201         }
202         Map<String, Object> cellMap = cellToStringMap(cell);
203         qualifierDetails.add(cellMap);
204       }
205     }
206     map.put("totalColumns", colCount);
207     // add the id if set
208     if (getId() != null) {
209       map.put("id", getId());
210     }
211     // Add the TTL if set
212     // Long.MAX_VALUE is the default, and is interpreted to mean this attribute
213     // has not been set.
214     if (getTTL() != Long.MAX_VALUE) {
215       map.put("ttl", getTTL());
216     }
217     return map;
218   }
219 
220   private static Map<String, Object> cellToStringMap(Cell c) {
221     Map<String, Object> stringMap = new HashMap<String, Object>();
222     stringMap.put("qualifier", Bytes.toStringBinary(c.getQualifierArray(), c.getQualifierOffset(),
223                 c.getQualifierLength()));
224     stringMap.put("timestamp", c.getTimestamp());
225     stringMap.put("vlen", c.getValueLength());
226     List<Tag> tags = Tag.asList(c.getTagsArray(), c.getTagsOffset(), c.getTagsLength());
227     if (tags != null) {
228       List<String> tagsString = new ArrayList<String>();
229       for (Tag t : tags) {
230         tagsString.add((t.getType()) + ":" + Bytes.toStringBinary(t.getValue()));
231       }
232       stringMap.put("tag", tagsString);
233     }
234     return stringMap;
235   }
236 
237   /**
238    * @deprecated Use {@link #getDurability()} instead.
239    * @return true if edits should be applied to WAL, false if not
240    */
241   @Deprecated
242   public boolean getWriteToWAL() {
243     return this.durability != Durability.SKIP_WAL;
244   }
245 
246   /**
247    * Set whether this Delete should be written to the WAL or not.
248    * Not writing the WAL means you may lose edits on server crash.
249    * This method will reset any changes made via {@link #setDurability(Durability)}
250    * @param write true if edits should be written to WAL, false if not
251    * @deprecated Use {@link #setDurability(Durability)} instead.
252    */
253   @Deprecated
254   public Mutation setWriteToWAL(boolean write) {
255     setDurability(write ? Durability.USE_DEFAULT : Durability.SKIP_WAL);
256     return this;
257   }
258 
259   /**
260    * Set the durability for this mutation
261    * @param d
262    */
263   public Mutation setDurability(Durability d) {
264     this.durability = d;
265     return this;
266   }
267 
268   /** Get the current durability */
269   public Durability getDurability() {
270     return this.durability;
271   }
272 
273   /**
274    * Method for retrieving the put's familyMap
275    * @return familyMap
276    */
277   public NavigableMap<byte [], List<Cell>> getFamilyCellMap() {
278     return this.familyMap;
279   }
280 
281   /**
282    * Method for setting the put's familyMap
283    */
284   public Mutation setFamilyCellMap(NavigableMap<byte [], List<Cell>> map) {
285     // TODO: Shut this down or move it up to be a Constructor.  Get new object rather than change
286     // this internal data member.
287     this.familyMap = map;
288     return this;
289   }
290 
291   /**
292    * Method for retrieving the put's familyMap that is deprecated and inefficient.
293    * @return the map
294    * @deprecated use {@link #getFamilyCellMap()} instead.
295    */
296   @Deprecated
297   public NavigableMap<byte [], List<KeyValue>> getFamilyMap() {
298     TreeMap<byte[], List<KeyValue>> fm =
299         new TreeMap<byte[], List<KeyValue>>(Bytes.BYTES_COMPARATOR);
300     for (Map.Entry<byte[], List<Cell>> e : familyMap.entrySet()) {
301       List<KeyValue> kvl = new ArrayList<KeyValue>(e.getValue().size());
302       for (Cell c : e.getValue()) {
303         kvl.add(KeyValueUtil.ensureKeyValue(c));
304       }
305       fm.put(e.getKey(), kvl);
306     }
307     return fm;
308   }
309 
310   /**
311    * Method for setting the put's familyMap that is deprecated and inefficient.
312    * @deprecated use {@link #setFamilyCellMap(NavigableMap)} instead.
313    */
314   @Deprecated
315   public Mutation setFamilyMap(NavigableMap<byte [], List<KeyValue>> map) {
316     TreeMap<byte[], List<Cell>> fm = new TreeMap<byte[], List<Cell>>(Bytes.BYTES_COMPARATOR);
317     for (Map.Entry<byte[], List<KeyValue>> e : map.entrySet()) {
318       fm.put(e.getKey(), Lists.<Cell>newArrayList(e.getValue()));
319     }
320     this.familyMap = fm;
321     return this;
322   }
323 
324   /**
325    * Method to check if the familyMap is empty
326    * @return true if empty, false otherwise
327    */
328   public boolean isEmpty() {
329     return familyMap.isEmpty();
330   }
331 
332   /**
333    * Method for retrieving the delete's row
334    * @return row
335    */
336   @Override
337   public byte [] getRow() {
338     return this.row;
339   }
340 
341   @Override
342   public int compareTo(final Row d) {
343     return Bytes.compareTo(this.getRow(), d.getRow());
344   }
345 
346   /**
347    * Method for retrieving the timestamp
348    * @return timestamp
349    */
350   public long getTimeStamp() {
351     return this.ts;
352   }
353 
354   /**
355    * Marks that the clusters with the given clusterIds have consumed the mutation
356    * @param clusterIds of the clusters that have consumed the mutation
357    */
358   public Mutation setClusterIds(List<UUID> clusterIds) {
359     ByteArrayDataOutput out = ByteStreams.newDataOutput();
360     out.writeInt(clusterIds.size());
361     for (UUID clusterId : clusterIds) {
362       out.writeLong(clusterId.getMostSignificantBits());
363       out.writeLong(clusterId.getLeastSignificantBits());
364     }
365     setAttribute(CONSUMED_CLUSTER_IDS, out.toByteArray());
366     return this;
367   }
368 
369   /**
370    * @return the set of clusterIds that have consumed the mutation
371    */
372   public List<UUID> getClusterIds() {
373     List<UUID> clusterIds = new ArrayList<UUID>();
374     byte[] bytes = getAttribute(CONSUMED_CLUSTER_IDS);
375     if(bytes != null) {
376       ByteArrayDataInput in = ByteStreams.newDataInput(bytes);
377       int numClusters = in.readInt();
378       for(int i=0; i<numClusters; i++){
379         clusterIds.add(new UUID(in.readLong(), in.readLong()));
380       }
381     }
382     return clusterIds;
383   }
384 
385   /**
386    * Sets the visibility expression associated with cells in this Mutation.
387    * It is illegal to set <code>CellVisibility</code> on <code>Delete</code> mutation.
388    * @param expression
389    */
390   public Mutation setCellVisibility(CellVisibility expression) {
391     this.setAttribute(VisibilityConstants.VISIBILITY_LABELS_ATTR_KEY, ProtobufUtil
392         .toCellVisibility(expression).toByteArray());
393     return this;
394   }
395 
396   /**
397    * @return CellVisibility associated with cells in this Mutation.
398    * @throws DeserializationException
399    */
400   public CellVisibility getCellVisibility() throws DeserializationException {
401     byte[] cellVisibilityBytes = this.getAttribute(VisibilityConstants.VISIBILITY_LABELS_ATTR_KEY);
402     if (cellVisibilityBytes == null) return null;
403     return ProtobufUtil.toCellVisibility(cellVisibilityBytes);
404   }
405 
406   /**
407    * Number of KeyValues carried by this Mutation.
408    * @return the total number of KeyValues
409    */
410   public int size() {
411     int size = 0;
412     for (List<Cell> cells : this.familyMap.values()) {
413       size += cells.size();
414     }
415     return size;
416   }
417 
418   /**
419    * @return the number of different families
420    */
421   public int numFamilies() {
422     return familyMap.size();
423   }
424 
425   /**
426    * @return Calculate what Mutation adds to class heap size.
427    */
428   @Override
429   public long heapSize() {
430     long heapsize = MUTATION_OVERHEAD;
431     // Adding row
432     heapsize += ClassSize.align(ClassSize.ARRAY + this.row.length);
433 
434     // Adding map overhead
435     heapsize +=
436       ClassSize.align(this.familyMap.size() * ClassSize.MAP_ENTRY);
437     for(Map.Entry<byte [], List<Cell>> entry : this.familyMap.entrySet()) {
438       //Adding key overhead
439       heapsize +=
440         ClassSize.align(ClassSize.ARRAY + entry.getKey().length);
441 
442       //This part is kinds tricky since the JVM can reuse references if you
443       //store the same value, but have a good match with SizeOf at the moment
444       //Adding value overhead
445       heapsize += ClassSize.align(ClassSize.ARRAYLIST);
446       int size = entry.getValue().size();
447       heapsize += ClassSize.align(ClassSize.ARRAY +
448           size * ClassSize.REFERENCE);
449 
450       for(Cell cell : entry.getValue()) {
451         heapsize += CellUtil.estimatedHeapSizeOf(cell);
452       }
453     }
454     heapsize += getAttributeSize();
455     heapsize += extraHeapSize();
456     return ClassSize.align(heapsize);
457   }
458 
459   /**
460    * @return The serialized ACL for this operation, or null if none
461    */
462   public byte[] getACL() {
463     return getAttribute(AccessControlConstants.OP_ATTRIBUTE_ACL);
464   }
465 
466   /**
467    * @param user User short name
468    * @param perms Permissions for the user
469    */
470   public Mutation setACL(String user, Permission perms) {
471     setAttribute(AccessControlConstants.OP_ATTRIBUTE_ACL,
472       ProtobufUtil.toUsersAndPermissions(user, perms).toByteArray());
473     return this;
474   }
475 
476   /**
477    * @param perms A map of permissions for a user or users
478    */
479   public Mutation setACL(Map<String, Permission> perms) {
480     ListMultimap<String, Permission> permMap = ArrayListMultimap.create();
481     for (Map.Entry<String, Permission> entry : perms.entrySet()) {
482       permMap.put(entry.getKey(), entry.getValue());
483     }
484     setAttribute(AccessControlConstants.OP_ATTRIBUTE_ACL,
485       ProtobufUtil.toUsersAndPermissions(permMap).toByteArray());
486     return this;
487   }
488 
489   /**
490    * Return the TTL requested for the result of the mutation, in milliseconds.
491    * @return the TTL requested for the result of the mutation, in milliseconds,
492    * or Long.MAX_VALUE if unset
493    */
494   public long getTTL() {
495     byte[] ttlBytes = getAttribute(OP_ATTRIBUTE_TTL);
496     if (ttlBytes != null) {
497       return Bytes.toLong(ttlBytes);
498     }
499     return Long.MAX_VALUE;
500   }
501 
502   /**
503    * Set the TTL desired for the result of the mutation, in milliseconds.
504    * @param ttl the TTL desired for the result of the mutation, in milliseconds
505    * @return this
506    */
507   public Mutation setTTL(long ttl) {
508     setAttribute(OP_ATTRIBUTE_TTL, Bytes.toBytes(ttl));
509     return this;
510   }
511 
512   /**
513    * Subclasses should override this method to add the heap size of their own fields.
514    * @return the heap size to add (will be aligned).
515    */
516   protected long extraHeapSize(){
517     return 0L;
518   }
519 
520 
521   /**
522    * @param row Row to check
523    * @throws IllegalArgumentException Thrown if <code>row</code> is empty or null or
524    * &gt; {@link HConstants#MAX_ROW_LENGTH}
525    * @return <code>row</code>
526    */
527   static byte [] checkRow(final byte [] row) {
528     return checkRow(row, 0, row == null? 0: row.length);
529   }
530 
531   /**
532    * @param row Row to check
533    * @param offset
534    * @param length
535    * @throws IllegalArgumentException Thrown if <code>row</code> is empty or null or
536    * &gt; {@link HConstants#MAX_ROW_LENGTH}
537    * @return <code>row</code>
538    */
539   static byte [] checkRow(final byte [] row, final int offset, final int length) {
540     if (row == null) {
541       throw new IllegalArgumentException("Row buffer is null");
542     }
543     if (length == 0) {
544       throw new IllegalArgumentException("Row length is 0");
545     }
546     if (length > HConstants.MAX_ROW_LENGTH) {
547       throw new IllegalArgumentException("Row length " + length + " is > " +
548         HConstants.MAX_ROW_LENGTH);
549     }
550     return row;
551   }
552 
553   static void checkRow(ByteBuffer row) {
554     if (row == null) {
555       throw new IllegalArgumentException("Row buffer is null");
556     }
557     if (row.remaining() == 0) {
558       throw new IllegalArgumentException("Row length is 0");
559     }
560     if (row.remaining() > HConstants.MAX_ROW_LENGTH) {
561       throw new IllegalArgumentException("Row length " + row.remaining() + " is > " +
562           HConstants.MAX_ROW_LENGTH);
563     }
564   }
565 }