View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.client;
20  
21  import java.nio.ByteBuffer;
22  import java.util.ArrayList;
23  import java.util.Arrays;
24  import java.util.HashMap;
25  import java.util.List;
26  import java.util.Map;
27  import java.util.NavigableMap;
28  import java.util.TreeMap;
29  import java.util.UUID;
30  
31  import org.apache.hadoop.classification.InterfaceAudience;
32  import org.apache.hadoop.classification.InterfaceStability;
33  import org.apache.hadoop.hbase.Cell;
34  import org.apache.hadoop.hbase.CellScannable;
35  import org.apache.hadoop.hbase.CellScanner;
36  import org.apache.hadoop.hbase.CellUtil;
37  import org.apache.hadoop.hbase.HConstants;
38  import org.apache.hadoop.hbase.KeyValue;
39  import org.apache.hadoop.hbase.KeyValueUtil;
40  import org.apache.hadoop.hbase.Tag;
41  import org.apache.hadoop.hbase.exceptions.DeserializationException;
42  import org.apache.hadoop.hbase.io.HeapSize;
43  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
44  import org.apache.hadoop.hbase.security.access.AccessControlConstants;
45  import org.apache.hadoop.hbase.security.access.Permission;
46  import org.apache.hadoop.hbase.security.visibility.CellVisibility;
47  import org.apache.hadoop.hbase.security.visibility.VisibilityConstants;
48  import org.apache.hadoop.hbase.util.Bytes;
49  import org.apache.hadoop.hbase.util.ClassSize;
50  
51  import com.google.common.collect.ArrayListMultimap;
52  import com.google.common.collect.ListMultimap;
53  import com.google.common.collect.Lists;
54  import com.google.common.io.ByteArrayDataInput;
55  import com.google.common.io.ByteArrayDataOutput;
56  import com.google.common.io.ByteStreams;
57  
58  @InterfaceAudience.Public
59  @InterfaceStability.Evolving
60  public abstract class Mutation extends OperationWithAttributes implements Row, CellScannable,
61      HeapSize {
62    public static final long MUTATION_OVERHEAD = ClassSize.align(
63        // This
64        ClassSize.OBJECT +
65        // row + OperationWithAttributes.attributes
66        2 * ClassSize.REFERENCE +
67        // Timestamp
68        1 * Bytes.SIZEOF_LONG +
69        // durability
70        ClassSize.REFERENCE +
71        // familyMap
72        ClassSize.REFERENCE +
73        // familyMap
74        ClassSize.TREEMAP);
75  
76    /**
77     * The attribute for storing the list of clusters that have consumed the change.
78     */
79    private static final String CONSUMED_CLUSTER_IDS = "_cs.id";
80  
81    protected byte [] row = null;
82    protected long ts = HConstants.LATEST_TIMESTAMP;
83    protected Durability durability = Durability.USE_DEFAULT;
84  
85    // A Map sorted by column family.
86    protected NavigableMap<byte [], List<Cell>> familyMap =
87      new TreeMap<byte [], List<Cell>>(Bytes.BYTES_COMPARATOR);
88  
89    @Override
90    public CellScanner cellScanner() {
91      return CellUtil.createCellScanner(getFamilyCellMap());
92    }
93  
94    /**
95     * Creates an empty list if one doesn't exist for the given column family
96     * or else it returns the associated list of Cell objects.
97     *
98     * @param family column family
99     * @return a list of Cell objects, returns an empty list if one doesn't exist.
100    */
101   List<Cell> getCellList(byte[] family) {
102     List<Cell> list = this.familyMap.get(family);
103     if (list == null) {
104       list = new ArrayList<Cell>();
105     }
106     return list;
107   }
108 
109   /*
110    * Create a KeyValue with this objects row key and the Put identifier.
111    *
112    * @return a KeyValue with this objects row key and the Put identifier.
113    */
114   KeyValue createPutKeyValue(byte[] family, byte[] qualifier, long ts, byte[] value) {
115     return new KeyValue(this.row, family, qualifier, ts, KeyValue.Type.Put, value);
116   }
117 
118   /**
119    * Create a KeyValue with this objects row key and the Put identifier.
120    * @param family
121    * @param qualifier
122    * @param ts
123    * @param value
124    * @param tags - Specify the Tags as an Array {@link KeyValue.Tag}
125    * @return a KeyValue with this objects row key and the Put identifier.
126    */
127   KeyValue createPutKeyValue(byte[] family, byte[] qualifier, long ts, byte[] value, Tag[] tags) {
128     KeyValue kvWithTag = new KeyValue(this.row, family, qualifier, ts, value, tags);
129     return kvWithTag;
130   }
131 
132   /*
133    * Create a KeyValue with this objects row key and the Put identifier.
134    *
135    * @return a KeyValue with this objects row key and the Put identifier.
136    */
137   KeyValue createPutKeyValue(byte[] family, ByteBuffer qualifier, long ts, ByteBuffer value,
138                              Tag[] tags) {
139     return new KeyValue(this.row, 0, this.row == null ? 0 : this.row.length,
140         family, 0, family == null ? 0 : family.length,
141         qualifier, ts, KeyValue.Type.Put, value, tags != null ? Arrays.asList(tags) : null);
142   }
143 
144   /**
145    * Compile the column family (i.e. schema) information
146    * into a Map. Useful for parsing and aggregation by debugging,
147    * logging, and administration tools.
148    * @return Map
149    */
150   @Override
151   public Map<String, Object> getFingerprint() {
152     Map<String, Object> map = new HashMap<String, Object>();
153     List<String> families = new ArrayList<String>();
154     // ideally, we would also include table information, but that information
155     // is not stored in each Operation instance.
156     map.put("families", families);
157     for (Map.Entry<byte [], List<Cell>> entry : this.familyMap.entrySet()) {
158       families.add(Bytes.toStringBinary(entry.getKey()));
159     }
160     return map;
161   }
162 
163   /**
164    * Compile the details beyond the scope of getFingerprint (row, columns,
165    * timestamps, etc.) into a Map along with the fingerprinted information.
166    * Useful for debugging, logging, and administration tools.
167    * @param maxCols a limit on the number of columns output prior to truncation
168    * @return Map
169    */
170   @Override
171   public Map<String, Object> toMap(int maxCols) {
172     // we start with the fingerprint map and build on top of it.
173     Map<String, Object> map = getFingerprint();
174     // replace the fingerprint's simple list of families with a
175     // map from column families to lists of qualifiers and kv details
176     Map<String, List<Map<String, Object>>> columns =
177       new HashMap<String, List<Map<String, Object>>>();
178     map.put("families", columns);
179     map.put("row", Bytes.toStringBinary(this.row));
180     int colCount = 0;
181     // iterate through all column families affected
182     for (Map.Entry<byte [], List<Cell>> entry : this.familyMap.entrySet()) {
183       // map from this family to details for each cell affected within the family
184       List<Map<String, Object>> qualifierDetails = new ArrayList<Map<String, Object>>();
185       columns.put(Bytes.toStringBinary(entry.getKey()), qualifierDetails);
186       colCount += entry.getValue().size();
187       if (maxCols <= 0) {
188         continue;
189       }
190       // add details for each cell
191       for (Cell cell: entry.getValue()) {
192         if (--maxCols <= 0 ) {
193           continue;
194         }
195         // KeyValue v1 expectation.  Cast for now until we go all Cell all the time.
196         KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
197         Map<String, Object> kvMap = kv.toStringMap();
198         // row and family information are already available in the bigger map
199         kvMap.remove("row");
200         kvMap.remove("family");
201         qualifierDetails.add(kvMap);
202       }
203     }
204     map.put("totalColumns", colCount);
205     // add the id if set
206     if (getId() != null) {
207       map.put("id", getId());
208     }
209     return map;
210   }
211 
212   /**
213    * @deprecated Use {@link #getDurability()} instead.
214    * @return true if edits should be applied to WAL, false if not
215    */
216   @Deprecated
217   public boolean getWriteToWAL() {
218     return this.durability == Durability.SKIP_WAL;
219   }
220 
221   /**
222    * Set whether this Delete should be written to the WAL or not.
223    * Not writing the WAL means you may lose edits on server crash.
224    * This method will reset any changes made via {@link #setDurability(Durability)}
225    * @param write true if edits should be written to WAL, false if not
226    * @deprecated Use {@link #setDurability(Durability)} instead.
227    */
228   @Deprecated
229   public void setWriteToWAL(boolean write) {
230     setDurability(write ? Durability.USE_DEFAULT : Durability.SKIP_WAL);
231   }
232 
233   /**
234    * Set the durability for this mutation
235    * @param d
236    */
237   public void setDurability(Durability d) {
238     this.durability = d;
239   }
240 
241   /** Get the current durability */
242   public Durability getDurability() {
243     return this.durability;
244   }
245 
246   /**
247    * Method for retrieving the put's familyMap
248    * @return familyMap
249    */
250   public NavigableMap<byte [], List<Cell>> getFamilyCellMap() {
251     return this.familyMap;
252   }
253 
254   /**
255    * Method for setting the put's familyMap
256    */
257   public void setFamilyCellMap(NavigableMap<byte [], List<Cell>> map) {
258     // TODO: Shut this down or move it up to be a Constructor.  Get new object rather than change
259     // this internal data member.
260     this.familyMap = map;
261   }
262 
263   /**
264    * Method for retrieving the put's familyMap that is deprecated and inefficient.
265    * @return the map
266    * @deprecated use {@link #getFamilyCellMap()} instead.
267    */
268   @Deprecated
269   public NavigableMap<byte [], List<KeyValue>> getFamilyMap() {
270     TreeMap<byte[], List<KeyValue>> fm =
271         new TreeMap<byte[], List<KeyValue>>(Bytes.BYTES_COMPARATOR);
272     for (Map.Entry<byte[], List<Cell>> e : familyMap.entrySet()) {
273       List<KeyValue> kvl = new ArrayList<KeyValue>(e.getValue().size());
274       for (Cell c : e.getValue()) {
275         kvl.add(KeyValueUtil.ensureKeyValue(c));
276       }
277       fm.put(e.getKey(), kvl);
278     }
279     return fm;
280   }
281 
282   /**
283    * Method for setting the put's familyMap that is deprecated and inefficient.
284    * @deprecated use {@link #setFamilyCellMap(NavigableMap)} instead.
285    */
286   @Deprecated
287   public void setFamilyMap(NavigableMap<byte [], List<KeyValue>> map) {
288     TreeMap<byte[], List<Cell>> fm = new TreeMap<byte[], List<Cell>>(Bytes.BYTES_COMPARATOR);
289     for (Map.Entry<byte[], List<KeyValue>> e : map.entrySet()) {
290       fm.put(e.getKey(), Lists.<Cell>newArrayList(e.getValue()));
291     }
292     this.familyMap = fm;
293   }
294 
295   /**
296    * Method to check if the familyMap is empty
297    * @return true if empty, false otherwise
298    */
299   public boolean isEmpty() {
300     return familyMap.isEmpty();
301   }
302 
303   /**
304    * Method for retrieving the delete's row
305    * @return row
306    */
307   @Override
308   public byte [] getRow() {
309     return this.row;
310   }
311 
312   @Override
313   public int compareTo(final Row d) {
314     return Bytes.compareTo(this.getRow(), d.getRow());
315   }
316 
317   /**
318    * Method for retrieving the timestamp
319    * @return timestamp
320    */
321   public long getTimeStamp() {
322     return this.ts;
323   }
324 
325   /**
326    * Marks that the clusters with the given clusterIds have consumed the mutation
327    * @param clusterIds of the clusters that have consumed the mutation
328    */
329   public void setClusterIds(List<UUID> clusterIds) {
330     ByteArrayDataOutput out = ByteStreams.newDataOutput();
331     out.writeInt(clusterIds.size());
332     for (UUID clusterId : clusterIds) {
333       out.writeLong(clusterId.getMostSignificantBits());
334       out.writeLong(clusterId.getLeastSignificantBits());
335     }
336     setAttribute(CONSUMED_CLUSTER_IDS, out.toByteArray());
337   }
338 
339   /**
340    * @return the set of clusterIds that have consumed the mutation
341    */
342   public List<UUID> getClusterIds() {
343     List<UUID> clusterIds = new ArrayList<UUID>();
344     byte[] bytes = getAttribute(CONSUMED_CLUSTER_IDS);
345     if(bytes != null) {
346       ByteArrayDataInput in = ByteStreams.newDataInput(bytes);
347       int numClusters = in.readInt();
348       for(int i=0; i<numClusters; i++){
349         clusterIds.add(new UUID(in.readLong(), in.readLong()));
350       }
351     }
352     return clusterIds;
353   }
354 
355   /**
356    * Sets the visibility expression associated with cells in this Mutation.
357    * It is illegal to set <code>CellVisibility</code> on <code>Delete</code> mutation.
358    * @param expression
359    */
360   public void setCellVisibility(CellVisibility expression) {
361     this.setAttribute(VisibilityConstants.VISIBILITY_LABELS_ATTR_KEY, ProtobufUtil
362         .toCellVisibility(expression).toByteArray());
363   }
364 
365   /**
366    * @return CellVisibility associated with cells in this Mutation.
367    * @throws DeserializationException
368    */
369   public CellVisibility getCellVisibility() throws DeserializationException {
370     byte[] cellVisibilityBytes = this.getAttribute(VisibilityConstants.VISIBILITY_LABELS_ATTR_KEY);
371     if (cellVisibilityBytes == null) return null;
372     return ProtobufUtil.toCellVisibility(cellVisibilityBytes);
373   }
374 
375   /**
376    * Number of KeyValues carried by this Mutation.
377    * @return the total number of KeyValues
378    */
379   public int size() {
380     int size = 0;
381     for (List<Cell> cells : this.familyMap.values()) {
382       size += cells.size();
383     }
384     return size;
385   }
386 
387   /**
388    * @return the number of different families
389    */
390   public int numFamilies() {
391     return familyMap.size();
392   }
393 
394   /**
395    * @return Calculate what Mutation adds to class heap size.
396    */
397   @Override
398   public long heapSize() {
399     long heapsize = MUTATION_OVERHEAD;
400     // Adding row
401     heapsize += ClassSize.align(ClassSize.ARRAY + this.row.length);
402 
403     // Adding map overhead
404     heapsize +=
405       ClassSize.align(this.familyMap.size() * ClassSize.MAP_ENTRY);
406     for(Map.Entry<byte [], List<Cell>> entry : this.familyMap.entrySet()) {
407       //Adding key overhead
408       heapsize +=
409         ClassSize.align(ClassSize.ARRAY + entry.getKey().length);
410 
411       //This part is kinds tricky since the JVM can reuse references if you
412       //store the same value, but have a good match with SizeOf at the moment
413       //Adding value overhead
414       heapsize += ClassSize.align(ClassSize.ARRAYLIST);
415       int size = entry.getValue().size();
416       heapsize += ClassSize.align(ClassSize.ARRAY +
417           size * ClassSize.REFERENCE);
418 
419       for(Cell cell : entry.getValue()) {
420         KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
421         heapsize += kv.heapSize();
422       }
423     }
424     heapsize += getAttributeSize();
425     heapsize += extraHeapSize();
426     return ClassSize.align(heapsize);
427   }
428 
429   /**
430    * @return The serialized ACL for this operation, or null if none
431    */
432   public byte[] getACL() {
433     return getAttribute(AccessControlConstants.OP_ATTRIBUTE_ACL);
434   }
435 
436   /**
437    * @param user User short name
438    * @param perms Permissions for the user
439    */
440   public void setACL(String user, Permission perms) {
441     setAttribute(AccessControlConstants.OP_ATTRIBUTE_ACL,
442       ProtobufUtil.toUsersAndPermissions(user, perms).toByteArray());
443   }
444 
445   /**
446    * @param perms A map of permissions for a user or users
447    */
448   public void setACL(Map<String, Permission> perms) {
449     ListMultimap<String, Permission> permMap = ArrayListMultimap.create();
450     for (Map.Entry<String, Permission> entry : perms.entrySet()) {
451       permMap.put(entry.getKey(), entry.getValue());
452     }
453     setAttribute(AccessControlConstants.OP_ATTRIBUTE_ACL,
454       ProtobufUtil.toUsersAndPermissions(permMap).toByteArray());
455   }
456 
457   /**
458    * @return true if ACLs should be evaluated on the cell level first
459    */
460   public boolean getACLStrategy() {
461     byte[] bytes = getAttribute(AccessControlConstants.OP_ATTRIBUTE_ACL_STRATEGY);
462     if (bytes != null) {
463       return Bytes.equals(bytes, AccessControlConstants.OP_ATTRIBUTE_ACL_STRATEGY_CELL_FIRST);
464     }
465     return false;
466   }
467 
468   /**
469    * @param cellFirstStrategy true if ACLs should be evaluated on the cell
470    * level first, false if ACL should first be checked at the CF and table
471    * levels
472    */
473   public void setACLStrategy(boolean cellFirstStrategy) {
474     if (cellFirstStrategy) {
475       setAttribute(AccessControlConstants.OP_ATTRIBUTE_ACL_STRATEGY,
476         AccessControlConstants.OP_ATTRIBUTE_ACL_STRATEGY_CELL_FIRST);
477     }
478   }
479 
480   /**
481    * Subclasses should override this method to add the heap size of their own fields.
482    * @return the heap size to add (will be aligned).
483    */
484   protected long extraHeapSize(){
485     return 0L;
486   }
487 
488 
489   /**
490    * @param row Row to check
491    * @throws IllegalArgumentException Thrown if <code>row</code> is empty or null or
492    * &gt; {@link HConstants#MAX_ROW_LENGTH}
493    * @return <code>row</code>
494    */
495   static byte [] checkRow(final byte [] row) {
496     return checkRow(row, 0, row == null? 0: row.length);
497   }
498 
499   /**
500    * @param row Row to check
501    * @param offset
502    * @param length
503    * @throws IllegalArgumentException Thrown if <code>row</code> is empty or null or
504    * &gt; {@link HConstants#MAX_ROW_LENGTH}
505    * @return <code>row</code>
506    */
507   static byte [] checkRow(final byte [] row, final int offset, final int length) {
508     if (row == null) {
509       throw new IllegalArgumentException("Row buffer is null");
510     }
511     if (length == 0) {
512       throw new IllegalArgumentException("Row length is 0");
513     }
514     if (length > HConstants.MAX_ROW_LENGTH) {
515       throw new IllegalArgumentException("Row length " + length + " is > " +
516         HConstants.MAX_ROW_LENGTH);
517     }
518     return row;
519   }
520 
521   static void checkRow(ByteBuffer row) {
522     if (row == null) {
523       throw new IllegalArgumentException("Row buffer is null");
524     }
525     if (row.remaining() == 0) {
526       throw new IllegalArgumentException("Row length is 0");
527     }
528     if (row.remaining() > HConstants.MAX_ROW_LENGTH) {
529       throw new IllegalArgumentException("Row length " + row.remaining() + " is > " +
530           HConstants.MAX_ROW_LENGTH);
531     }
532   }
533 }