View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.client;
20  
21  import java.nio.ByteBuffer;
22  import java.util.ArrayList;
23  import java.util.Arrays;
24  import java.util.HashMap;
25  import java.util.List;
26  import java.util.Map;
27  import java.util.NavigableMap;
28  import java.util.TreeMap;
29  import java.util.UUID;
30  
31  import org.apache.hadoop.hbase.Cell;
32  import org.apache.hadoop.hbase.CellScannable;
33  import org.apache.hadoop.hbase.CellScanner;
34  import org.apache.hadoop.hbase.CellUtil;
35  import org.apache.hadoop.hbase.HConstants;
36  import org.apache.hadoop.hbase.KeyValue;
37  import org.apache.hadoop.hbase.Tag;
38  import org.apache.hadoop.hbase.classification.InterfaceAudience;
39  import org.apache.hadoop.hbase.classification.InterfaceStability;
40  import org.apache.hadoop.hbase.exceptions.DeserializationException;
41  import org.apache.hadoop.hbase.io.HeapSize;
42  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
43  import org.apache.hadoop.hbase.security.access.AccessControlConstants;
44  import org.apache.hadoop.hbase.security.access.Permission;
45  import org.apache.hadoop.hbase.security.visibility.CellVisibility;
46  import org.apache.hadoop.hbase.security.visibility.VisibilityConstants;
47  import org.apache.hadoop.hbase.util.Bytes;
48  import org.apache.hadoop.hbase.util.ClassSize;
49  
50  import com.google.common.collect.ArrayListMultimap;
51  import com.google.common.collect.ListMultimap;
52  import com.google.common.io.ByteArrayDataInput;
53  import com.google.common.io.ByteArrayDataOutput;
54  import com.google.common.io.ByteStreams;
55  
56  @InterfaceAudience.Public
57  @InterfaceStability.Evolving
58  public abstract class Mutation extends OperationWithAttributes implements Row, CellScannable,
59      HeapSize {
60    public static final long MUTATION_OVERHEAD = ClassSize.align(
61        // This
62        ClassSize.OBJECT +
63        // row + OperationWithAttributes.attributes
64        2 * ClassSize.REFERENCE +
65        // Timestamp
66        1 * Bytes.SIZEOF_LONG +
67        // durability
68        ClassSize.REFERENCE +
69        // familyMap
70        ClassSize.REFERENCE +
71        // familyMap
72        ClassSize.TREEMAP);
73  
74    /**
75     * The attribute for storing the list of clusters that have consumed the change.
76     */
77    private static final String CONSUMED_CLUSTER_IDS = "_cs.id";
78  
79    protected byte [] row = null;
80    protected long ts = HConstants.LATEST_TIMESTAMP;
81    protected Durability durability = Durability.USE_DEFAULT;
82  
83    // A Map sorted by column family.
84    protected NavigableMap<byte [], List<Cell>> familyMap =
85      new TreeMap<byte [], List<Cell>>(Bytes.BYTES_COMPARATOR);
86  
87    @Override
88    public CellScanner cellScanner() {
89      return CellUtil.createCellScanner(getFamilyCellMap());
90    }
91  
92    /**
93     * Creates an empty list if one doesn't exist for the given column family
94     * or else it returns the associated list of Cell objects.
95     *
96     * @param family column family
97     * @return a list of Cell objects, returns an empty list if one doesn't exist.
98     */
99    List<Cell> getCellList(byte[] family) {
100     List<Cell> list = this.familyMap.get(family);
101     if (list == null) {
102       list = new ArrayList<Cell>();
103     }
104     return list;
105   }
106 
107   /*
108    * Create a KeyValue with this objects row key and the Put identifier.
109    *
110    * @return a KeyValue with this objects row key and the Put identifier.
111    */
112   KeyValue createPutKeyValue(byte[] family, byte[] qualifier, long ts, byte[] value) {
113     return new KeyValue(this.row, family, qualifier, ts, KeyValue.Type.Put, value);
114   }
115 
116   /**
117    * Create a KeyValue with this objects row key and the Put identifier.
118    * @param family
119    * @param qualifier
120    * @param ts
121    * @param value
122    * @param tags - Specify the Tags as an Array {@link KeyValue.Tag}
123    * @return a KeyValue with this objects row key and the Put identifier.
124    */
125   KeyValue createPutKeyValue(byte[] family, byte[] qualifier, long ts, byte[] value, Tag[] tags) {
126     KeyValue kvWithTag = new KeyValue(this.row, family, qualifier, ts, value, tags);
127     return kvWithTag;
128   }
129 
130   /*
131    * Create a KeyValue with this objects row key and the Put identifier.
132    *
133    * @return a KeyValue with this objects row key and the Put identifier.
134    */
135   KeyValue createPutKeyValue(byte[] family, ByteBuffer qualifier, long ts, ByteBuffer value,
136                              Tag[] tags) {
137     return new KeyValue(this.row, 0, this.row == null ? 0 : this.row.length,
138         family, 0, family == null ? 0 : family.length,
139         qualifier, ts, KeyValue.Type.Put, value, tags != null ? Arrays.asList(tags) : null);
140   }
141 
142   /**
143    * Compile the column family (i.e. schema) information
144    * into a Map. Useful for parsing and aggregation by debugging,
145    * logging, and administration tools.
146    * @return Map
147    */
148   @Override
149   public Map<String, Object> getFingerprint() {
150     Map<String, Object> map = new HashMap<String, Object>();
151     List<String> families = new ArrayList<String>();
152     // ideally, we would also include table information, but that information
153     // is not stored in each Operation instance.
154     map.put("families", families);
155     for (Map.Entry<byte [], List<Cell>> entry : this.familyMap.entrySet()) {
156       families.add(Bytes.toStringBinary(entry.getKey()));
157     }
158     return map;
159   }
160 
161   /**
162    * Compile the details beyond the scope of getFingerprint (row, columns,
163    * timestamps, etc.) into a Map along with the fingerprinted information.
164    * Useful for debugging, logging, and administration tools.
165    * @param maxCols a limit on the number of columns output prior to truncation
166    * @return Map
167    */
168   @Override
169   public Map<String, Object> toMap(int maxCols) {
170     // we start with the fingerprint map and build on top of it.
171     Map<String, Object> map = getFingerprint();
172     // replace the fingerprint's simple list of families with a
173     // map from column families to lists of qualifiers and kv details
174     Map<String, List<Map<String, Object>>> columns =
175       new HashMap<String, List<Map<String, Object>>>();
176     map.put("families", columns);
177     map.put("row", Bytes.toStringBinary(this.row));
178     int colCount = 0;
179     // iterate through all column families affected
180     for (Map.Entry<byte [], List<Cell>> entry : this.familyMap.entrySet()) {
181       // map from this family to details for each cell affected within the family
182       List<Map<String, Object>> qualifierDetails = new ArrayList<Map<String, Object>>();
183       columns.put(Bytes.toStringBinary(entry.getKey()), qualifierDetails);
184       colCount += entry.getValue().size();
185       if (maxCols <= 0) {
186         continue;
187       }
188       // add details for each cell
189       for (Cell cell: entry.getValue()) {
190         if (--maxCols <= 0 ) {
191           continue;
192         }
193         Map<String, Object> cellMap = cellToStringMap(cell);
194         qualifierDetails.add(cellMap);
195       }
196     }
197     map.put("totalColumns", colCount);
198     // add the id if set
199     if (getId() != null) {
200       map.put("id", getId());
201     }
202     return map;
203   }
204 
205   private static Map<String, Object> cellToStringMap(Cell c) {
206     Map<String, Object> stringMap = new HashMap<String, Object>();
207     stringMap.put("qualifier", Bytes.toStringBinary(c.getQualifierArray(), c.getQualifierOffset(),
208                 c.getQualifierLength()));
209     stringMap.put("timestamp", c.getTimestamp());
210     stringMap.put("vlen", c.getValueLength());
211     List<Tag> tags = Tag.asList(c.getTagsArray(), c.getTagsOffset(), c.getTagsLength());
212     if (tags != null) {
213       List<String> tagsString = new ArrayList<String>();
214       for (Tag t : tags) {
215         tagsString.add((t.getType()) + ":" + Bytes.toStringBinary(t.getValue()));
216       }
217       stringMap.put("tag", tagsString);
218     }
219     return stringMap;
220   }
221 
222   /**
223    * Set the durability for this mutation
224    * @param d
225    */
226   public Mutation setDurability(Durability d) {
227     this.durability = d;
228     return this;
229   }
230 
231   /** Get the current durability */
232   public Durability getDurability() {
233     return this.durability;
234   }
235 
236   /**
237    * Method for retrieving the put's familyMap
238    * @return familyMap
239    */
240   public NavigableMap<byte [], List<Cell>> getFamilyCellMap() {
241     return this.familyMap;
242   }
243 
244   /**
245    * Method for setting the put's familyMap
246    */
247   public Mutation setFamilyCellMap(NavigableMap<byte [], List<Cell>> map) {
248     // TODO: Shut this down or move it up to be a Constructor.  Get new object rather than change
249     // this internal data member.
250     this.familyMap = map;
251     return this;
252   }
253 
254   /**
255    * Method to check if the familyMap is empty
256    * @return true if empty, false otherwise
257    */
258   public boolean isEmpty() {
259     return familyMap.isEmpty();
260   }
261 
262   /**
263    * Method for retrieving the delete's row
264    * @return row
265    */
266   @Override
267   public byte [] getRow() {
268     return this.row;
269   }
270 
271   @Override
272   public int compareTo(final Row d) {
273     return Bytes.compareTo(this.getRow(), d.getRow());
274   }
275 
276   /**
277    * Method for retrieving the timestamp
278    * @return timestamp
279    */
280   public long getTimeStamp() {
281     return this.ts;
282   }
283 
284   /**
285    * Marks that the clusters with the given clusterIds have consumed the mutation
286    * @param clusterIds of the clusters that have consumed the mutation
287    */
288   public Mutation setClusterIds(List<UUID> clusterIds) {
289     ByteArrayDataOutput out = ByteStreams.newDataOutput();
290     out.writeInt(clusterIds.size());
291     for (UUID clusterId : clusterIds) {
292       out.writeLong(clusterId.getMostSignificantBits());
293       out.writeLong(clusterId.getLeastSignificantBits());
294     }
295     setAttribute(CONSUMED_CLUSTER_IDS, out.toByteArray());
296     return this;
297   }
298 
299   /**
300    * @return the set of clusterIds that have consumed the mutation
301    */
302   public List<UUID> getClusterIds() {
303     List<UUID> clusterIds = new ArrayList<UUID>();
304     byte[] bytes = getAttribute(CONSUMED_CLUSTER_IDS);
305     if(bytes != null) {
306       ByteArrayDataInput in = ByteStreams.newDataInput(bytes);
307       int numClusters = in.readInt();
308       for(int i=0; i<numClusters; i++){
309         clusterIds.add(new UUID(in.readLong(), in.readLong()));
310       }
311     }
312     return clusterIds;
313   }
314 
315   /**
316    * Sets the visibility expression associated with cells in this Mutation.
317    * It is illegal to set <code>CellVisibility</code> on <code>Delete</code> mutation.
318    * @param expression
319    */
320   public Mutation setCellVisibility(CellVisibility expression) {
321     this.setAttribute(VisibilityConstants.VISIBILITY_LABELS_ATTR_KEY, ProtobufUtil
322         .toCellVisibility(expression).toByteArray());
323     return this;
324   }
325 
326   /**
327    * @return CellVisibility associated with cells in this Mutation.
328    * @throws DeserializationException
329    */
330   public CellVisibility getCellVisibility() throws DeserializationException {
331     byte[] cellVisibilityBytes = this.getAttribute(VisibilityConstants.VISIBILITY_LABELS_ATTR_KEY);
332     if (cellVisibilityBytes == null) return null;
333     return ProtobufUtil.toCellVisibility(cellVisibilityBytes);
334   }
335 
336   /**
337    * Number of KeyValues carried by this Mutation.
338    * @return the total number of KeyValues
339    */
340   public int size() {
341     int size = 0;
342     for (List<Cell> cells : this.familyMap.values()) {
343       size += cells.size();
344     }
345     return size;
346   }
347 
348   /**
349    * @return the number of different families
350    */
351   public int numFamilies() {
352     return familyMap.size();
353   }
354 
355   /**
356    * @return Calculate what Mutation adds to class heap size.
357    */
358   @Override
359   public long heapSize() {
360     long heapsize = MUTATION_OVERHEAD;
361     // Adding row
362     heapsize += ClassSize.align(ClassSize.ARRAY + this.row.length);
363 
364     // Adding map overhead
365     heapsize +=
366       ClassSize.align(this.familyMap.size() * ClassSize.MAP_ENTRY);
367     for(Map.Entry<byte [], List<Cell>> entry : this.familyMap.entrySet()) {
368       //Adding key overhead
369       heapsize +=
370         ClassSize.align(ClassSize.ARRAY + entry.getKey().length);
371 
372       //This part is kinds tricky since the JVM can reuse references if you
373       //store the same value, but have a good match with SizeOf at the moment
374       //Adding value overhead
375       heapsize += ClassSize.align(ClassSize.ARRAYLIST);
376       int size = entry.getValue().size();
377       heapsize += ClassSize.align(ClassSize.ARRAY +
378           size * ClassSize.REFERENCE);
379 
380       for(Cell cell : entry.getValue()) {
381         heapsize += CellUtil.estimatedHeapSizeOf(cell);
382       }
383     }
384     heapsize += getAttributeSize();
385     heapsize += extraHeapSize();
386     return ClassSize.align(heapsize);
387   }
388 
389   /**
390    * @return The serialized ACL for this operation, or null if none
391    */
392   public byte[] getACL() {
393     return getAttribute(AccessControlConstants.OP_ATTRIBUTE_ACL);
394   }
395 
396   /**
397    * @param user User short name
398    * @param perms Permissions for the user
399    */
400   public Mutation setACL(String user, Permission perms) {
401     setAttribute(AccessControlConstants.OP_ATTRIBUTE_ACL,
402       ProtobufUtil.toUsersAndPermissions(user, perms).toByteArray());
403     return this;
404   }
405 
406   /**
407    * @param perms A map of permissions for a user or users
408    */
409   public Mutation setACL(Map<String, Permission> perms) {
410     ListMultimap<String, Permission> permMap = ArrayListMultimap.create();
411     for (Map.Entry<String, Permission> entry : perms.entrySet()) {
412       permMap.put(entry.getKey(), entry.getValue());
413     }
414     setAttribute(AccessControlConstants.OP_ATTRIBUTE_ACL,
415       ProtobufUtil.toUsersAndPermissions(permMap).toByteArray());
416     return this;
417   }
418 
419   /**
420    * Subclasses should override this method to add the heap size of their own fields.
421    * @return the heap size to add (will be aligned).
422    */
423   protected long extraHeapSize(){
424     return 0L;
425   }
426 
427 
428   /**
429    * @param row Row to check
430    * @throws IllegalArgumentException Thrown if <code>row</code> is empty or null or
431    * &gt; {@link HConstants#MAX_ROW_LENGTH}
432    * @return <code>row</code>
433    */
434   static byte [] checkRow(final byte [] row) {
435     return checkRow(row, 0, row == null? 0: row.length);
436   }
437 
438   /**
439    * @param row Row to check
440    * @param offset
441    * @param length
442    * @throws IllegalArgumentException Thrown if <code>row</code> is empty or null or
443    * &gt; {@link HConstants#MAX_ROW_LENGTH}
444    * @return <code>row</code>
445    */
446   static byte [] checkRow(final byte [] row, final int offset, final int length) {
447     if (row == null) {
448       throw new IllegalArgumentException("Row buffer is null");
449     }
450     if (length == 0) {
451       throw new IllegalArgumentException("Row length is 0");
452     }
453     if (length > HConstants.MAX_ROW_LENGTH) {
454       throw new IllegalArgumentException("Row length " + length + " is > " +
455         HConstants.MAX_ROW_LENGTH);
456     }
457     return row;
458   }
459 
460   static void checkRow(ByteBuffer row) {
461     if (row == null) {
462       throw new IllegalArgumentException("Row buffer is null");
463     }
464     if (row.remaining() == 0) {
465       throw new IllegalArgumentException("Row length is 0");
466     }
467     if (row.remaining() > HConstants.MAX_ROW_LENGTH) {
468       throw new IllegalArgumentException("Row length " + row.remaining() + " is > " +
469           HConstants.MAX_ROW_LENGTH);
470     }
471   }
472 }