001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.wal;
019
020import java.io.IOException;
021import java.util.ArrayList;
022import java.util.List;
023import java.util.Map;
024import java.util.Set;
025import java.util.TreeSet;
026import org.apache.hadoop.hbase.Cell;
027import org.apache.hadoop.hbase.CellUtil;
028import org.apache.hadoop.hbase.HBaseInterfaceAudience;
029import org.apache.hadoop.hbase.KeyValue;
030import org.apache.hadoop.hbase.PrivateCellUtil;
031import org.apache.hadoop.hbase.client.RegionInfo;
032import org.apache.hadoop.hbase.codec.Codec;
033import org.apache.hadoop.hbase.io.HeapSize;
034import org.apache.hadoop.hbase.util.Bytes;
035import org.apache.hadoop.hbase.util.ClassSize;
036import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
037import org.apache.yetus.audience.InterfaceAudience;
038
039import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos;
040import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.CompactionDescriptor;
041import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.FlushDescriptor;
042import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.RegionEventDescriptor;
043
044/**
045 * Used in HBase's transaction log (WAL) to represent a collection of edits (Cell/KeyValue objects)
046 * that came in as a single transaction. All the edits for a given transaction are written out as a
047 * single record, in PB format, followed (optionally) by Cells written via the WALCellEncoder.
048 * <p>
049 * This class is LimitedPrivate for CPs to read-only. The {@link #add} methods are classified as
050 * private methods, not for use by CPs.
051 * </p>
052 * <p>
053 * A particular WALEdit 'type' is the 'meta' type used to mark key operational events in the WAL
054 * such as compaction, flush, or region open. These meta types do not traverse hbase memstores. They
055 * are edits made by the hbase system rather than edit data submitted by clients. They only show in
056 * the WAL. These 'Meta' types have not been formally specified (or made into an explicit class
057 * type). They evolved organically. HBASE-8457 suggests codifying a WALEdit 'type' by adding a type
058 * field to WALEdit that gets serialized into the WAL. TODO. Would have to work on the
059 * consumption-side. Reading WALs on replay we seem to consume a Cell-at-a-time rather than by
060 * WALEdit. We are already in the below going out of our way to figure particular types -- e.g. if a
061 * compaction, replay, or close meta Marker -- during normal processing so would make sense to do
062 * this. Current system is an awkward marking of Cell columnfamily as {@link #METAFAMILY} and then
063 * setting qualifier based off meta edit type. For replay-time where we read Cell-at-a-time, there
064 * are utility methods below for figuring meta type. See also
065 * {@link #createBulkLoadEvent(RegionInfo, WALProtos.BulkLoadDescriptor)}, etc., for where we create
066 * meta WALEdit instances.
067 * </p>
068 * <p>
069 * WALEdit will accumulate a Set of all column family names referenced by the Cells
070 * {@link #add(Cell)}'d. This is an optimization. Usually when loading a WALEdit, we have the column
071 * family name to-hand.. just shove it into the WALEdit if available. Doing this, we can save on a
072 * parse of each Cell to figure column family down the line when we go to add the WALEdit to the WAL
073 * file. See the hand-off in FSWALEntry Constructor.
074 * @see WALKey
075 */
076// TODO: Do not expose this class to Coprocessors. It has set methods. A CP might meddle.
077@InterfaceAudience.LimitedPrivate({ HBaseInterfaceAudience.REPLICATION,
078  HBaseInterfaceAudience.COPROC })
079public class WALEdit implements HeapSize {
080  // Below defines are for writing WALEdit 'meta' Cells..
081  // TODO: Get rid of this system of special 'meta' Cells. See HBASE-8457. It suggests
082  // adding a type to WALEdit itself for use denoting meta Edits and their types.
083  public static final byte[] METAFAMILY = Bytes.toBytes("METAFAMILY");
084
085  /**
086   * @deprecated Since 2.3.0. Not used.
087   */
088  @Deprecated
089  public static final byte[] METAROW = Bytes.toBytes("METAROW");
090
091  /**
092   * @deprecated Since 2.3.0. Make it protected, internal-use only. Use
093   *             {@link #isCompactionMarker(Cell)}
094   */
095  @Deprecated
096  @InterfaceAudience.Private
097  public static final byte[] COMPACTION = Bytes.toBytes("HBASE::COMPACTION");
098
099  /**
100   * @deprecated Since 2.3.0. Make it protected, internal-use only.
101   */
102  @Deprecated
103  @InterfaceAudience.Private
104  public static final byte[] FLUSH = Bytes.toBytes("HBASE::FLUSH");
105
106  /**
107   * Qualifier for region event meta 'Marker' WALEdits start with the {@link #REGION_EVENT_PREFIX}
108   * prefix ('HBASE::REGION_EVENT::'). After the prefix, we note the type of the event which we get
109   * from the RegionEventDescriptor protobuf instance type (A RegionEventDescriptor protobuf
110   * instance is written as the meta Marker Cell value). Adding a type suffix means we do not have
111   * to deserialize the protobuf to figure out what type of event this is.. .just read the qualifier
112   * suffix. For example, a close region event descriptor will have a qualifier of
113   * HBASE::REGION_EVENT::REGION_CLOSE. See WAL.proto and the EventType in RegionEventDescriptor
114   * protos for all possible event types.
115   */
116  private static final String REGION_EVENT_STR = "HBASE::REGION_EVENT";
117  private static final String REGION_EVENT_PREFIX_STR = REGION_EVENT_STR + "::";
118  private static final byte[] REGION_EVENT_PREFIX = Bytes.toBytes(REGION_EVENT_PREFIX_STR);
119
120  /**
121   * @deprecated Since 2.3.0. Remove. Not for external use. Not used.
122   */
123  @Deprecated
124  public static final byte[] REGION_EVENT = Bytes.toBytes(REGION_EVENT_STR);
125
126  /**
127   * We use this define figuring if we are carrying a close event.
128   */
129  private static final byte[] REGION_EVENT_CLOSE =
130    createRegionEventDescriptorQualifier(RegionEventDescriptor.EventType.REGION_CLOSE);
131
132  @InterfaceAudience.Private
133  public static final byte[] BULK_LOAD = Bytes.toBytes("HBASE::BULK_LOAD");
134
135  private final transient boolean replay;
136
137  private ArrayList<Cell> cells;
138
139  /**
140   * All the Cell families in <code>cells</code>. Updated by {@link #add(Cell)} and
141   * {@link #add(Map)}. This Set is passed to the FSWALEntry so it does not have to recalculate the
142   * Set of families in a transaction; makes for a bunch of CPU savings.
143   */
144  private Set<byte[]> families = null;
145
146  public WALEdit() {
147    this(1, false);
148  }
149
150  /**
151   * @deprecated since 2.0.1 and will be removed in 4.0.0. Use {@link #WALEdit(int, boolean)}
152   *             instead.
153   * @see #WALEdit(int, boolean)
154   * @see <a href="https://issues.apache.org/jira/browse/HBASE-20781">HBASE-20781</a>
155   */
156  @Deprecated
157  public WALEdit(boolean replay) {
158    this(1, replay);
159  }
160
161  /**
162   * @deprecated since 2.0.1 and will be removed in 4.0.0. Use {@link #WALEdit(int, boolean)}
163   *             instead.
164   * @see #WALEdit(int, boolean)
165   * @see <a href="https://issues.apache.org/jira/browse/HBASE-20781">HBASE-20781</a>
166   */
167  @Deprecated
168  public WALEdit(int cellCount) {
169    this(cellCount, false);
170  }
171
172  /**
173   * @param cellCount Pass so can pre-size the WALEdit. Optimization.
174   */
175  public WALEdit(int cellCount, boolean isReplay) {
176    this.replay = isReplay;
177    cells = new ArrayList<>(cellCount);
178  }
179
180  private Set<byte[]> getOrCreateFamilies() {
181    if (this.families == null) {
182      this.families = new TreeSet<>(Bytes.BYTES_COMPARATOR);
183    }
184    return this.families;
185  }
186
187  /**
188   * For use by FSWALEntry ONLY. An optimization.
189   * @return All families in {@link #getCells()}; may be null.
190   */
191  public Set<byte[]> getFamilies() {
192    return this.families;
193  }
194
195  /**
196   * @return True is <code>f</code> is {@link #METAFAMILY}
197   * @deprecated Since 2.3.0. Do not expose. Make protected.
198   */
199  @Deprecated
200  public static boolean isMetaEditFamily(final byte[] f) {
201    return Bytes.equals(METAFAMILY, f);
202  }
203
204  /**
205   * Replaying WALs can read Cell-at-a-time so need this method in those cases.
206   */
207  public static boolean isMetaEditFamily(Cell cell) {
208    return CellUtil.matchingFamily(cell, METAFAMILY);
209  }
210
211  /**
212   * @return True if this is a meta edit; has one edit only and its columnfamily is
213   *         {@link #METAFAMILY}.
214   */
215  public boolean isMetaEdit() {
216    return this.families != null && this.families.size() == 1 && this.families.contains(METAFAMILY);
217  }
218
219  /**
220   * @return True when current WALEdit is created by log replay. Replication skips WALEdits from
221   *         replay.
222   */
223  public boolean isReplay() {
224    return this.replay;
225  }
226
227  @InterfaceAudience.Private
228  public WALEdit add(Cell cell, byte[] family) {
229    getOrCreateFamilies().add(family);
230    return addCell(cell);
231  }
232
233  @InterfaceAudience.Private
234  public WALEdit add(Cell cell) {
235    // We clone Family each time we add a Cell. Expensive but safe. For CPU savings, use
236    // add(Map) or add(Cell, family).
237    return add(cell, CellUtil.cloneFamily(cell));
238  }
239
240  public boolean isEmpty() {
241    return cells.isEmpty();
242  }
243
244  public int size() {
245    return cells.size();
246  }
247
248  public ArrayList<Cell> getCells() {
249    return cells;
250  }
251
252  /**
253   * This is not thread safe. This will change the WALEdit and shouldn't be used unless you are sure
254   * that nothing else depends on the contents being immutable.
255   * @param cells the list of cells that this WALEdit now contains.
256   */
257  @InterfaceAudience.Private
258  // Used by replay.
259  public void setCells(ArrayList<Cell> cells) {
260    this.cells = cells;
261    this.families = null;
262  }
263
264  /**
265   * Reads WALEdit from cells.
266   * @param cellDecoder   Cell decoder.
267   * @param expectedCount Expected cell count.
268   * @return Number of KVs read.
269   */
270  public int readFromCells(Codec.Decoder cellDecoder, int expectedCount) throws IOException {
271    cells.clear();
272    cells.ensureCapacity(expectedCount);
273    while (cells.size() < expectedCount && cellDecoder.advance()) {
274      add(cellDecoder.current());
275    }
276    return cells.size();
277  }
278
279  @Override
280  public long heapSize() {
281    long ret = ClassSize.ARRAYLIST;
282    for (Cell cell : cells) {
283      ret += cell.heapSize();
284    }
285    return ret;
286  }
287
288  public long estimatedSerializedSizeOf() {
289    long ret = 0;
290    for (Cell cell : cells) {
291      ret += PrivateCellUtil.estimatedSerializedSizeOf(cell);
292    }
293    return ret;
294  }
295
296  @Override
297  public String toString() {
298    StringBuilder sb = new StringBuilder();
299
300    sb.append("[#edits: ").append(cells.size()).append(" = <");
301    for (Cell cell : cells) {
302      sb.append(cell);
303      sb.append("; ");
304    }
305    sb.append(">]");
306    return sb.toString();
307  }
308
309  public static WALEdit createFlushWALEdit(RegionInfo hri, FlushDescriptor f) {
310    KeyValue kv = new KeyValue(getRowForRegion(hri), METAFAMILY, FLUSH,
311      EnvironmentEdgeManager.currentTime(), f.toByteArray());
312    return new WALEdit().add(kv, METAFAMILY);
313  }
314
315  public static FlushDescriptor getFlushDescriptor(Cell cell) throws IOException {
316    return CellUtil.matchingColumn(cell, METAFAMILY, FLUSH)
317      ? FlushDescriptor.parseFrom(CellUtil.cloneValue(cell))
318      : null;
319  }
320
321  /**
322   * @return A meta Marker WALEdit that has a single Cell whose value is the passed in
323   *         <code>regionEventDesc</code> serialized and whose row is this region, columnfamily is
324   *         {@link #METAFAMILY} and qualifier is {@link #REGION_EVENT_PREFIX} +
325   *         {@link RegionEventDescriptor#getEventType()}; for example
326   *         HBASE::REGION_EVENT::REGION_CLOSE.
327   */
328  public static WALEdit createRegionEventWALEdit(RegionInfo hri,
329    RegionEventDescriptor regionEventDesc) {
330    return createRegionEventWALEdit(getRowForRegion(hri), regionEventDesc);
331  }
332
333  @InterfaceAudience.Private
334  public static WALEdit createRegionEventWALEdit(byte[] rowForRegion,
335    RegionEventDescriptor regionEventDesc) {
336    KeyValue kv = new KeyValue(rowForRegion, METAFAMILY,
337      createRegionEventDescriptorQualifier(regionEventDesc.getEventType()),
338      EnvironmentEdgeManager.currentTime(), regionEventDesc.toByteArray());
339    return new WALEdit().add(kv, METAFAMILY);
340  }
341
342  /**
343   * @return Cell qualifier for the passed in RegionEventDescriptor Type; e.g. we'll return
344   *         something like a byte array with HBASE::REGION_EVENT::REGION_OPEN in it.
345   */
346  @InterfaceAudience.Private
347  public static byte[] createRegionEventDescriptorQualifier(RegionEventDescriptor.EventType t) {
348    return Bytes.toBytes(REGION_EVENT_PREFIX_STR + t.toString());
349  }
350
351  /**
352   * Public so can be accessed from regionserver.wal package.
353   * @return True if this is a Marker Edit and it is a RegionClose type.
354   */
355  public boolean isRegionCloseMarker() {
356    return isMetaEdit() && PrivateCellUtil.matchingQualifier(this.cells.get(0), REGION_EVENT_CLOSE,
357      0, REGION_EVENT_CLOSE.length);
358  }
359
360  /**
361   * @return Returns a RegionEventDescriptor made by deserializing the content of the passed in
362   *         <code>cell</code>, IFF the <code>cell</code> is a RegionEventDescriptor type WALEdit.
363   */
364  public static RegionEventDescriptor getRegionEventDescriptor(Cell cell) throws IOException {
365    return CellUtil.matchingColumnFamilyAndQualifierPrefix(cell, METAFAMILY, REGION_EVENT_PREFIX)
366      ? RegionEventDescriptor.parseFrom(CellUtil.cloneValue(cell))
367      : null;
368  }
369
370  /** Returns A Marker WALEdit that has <code>c</code> serialized as its value */
371  public static WALEdit createCompaction(final RegionInfo hri, final CompactionDescriptor c) {
372    byte[] pbbytes = c.toByteArray();
373    KeyValue kv = new KeyValue(getRowForRegion(hri), METAFAMILY, COMPACTION,
374      EnvironmentEdgeManager.currentTime(), pbbytes);
375    return new WALEdit().add(kv, METAFAMILY); // replication scope null so this won't be replicated
376  }
377
378  public static byte[] getRowForRegion(RegionInfo hri) {
379    byte[] startKey = hri.getStartKey();
380    if (startKey.length == 0) {
381      // empty row key is not allowed in mutations because it is both the start key and the end key
382      // we return the smallest byte[] that is bigger (in lex comparison) than byte[0].
383      return new byte[] { 0 };
384    }
385    return startKey;
386  }
387
388  /**
389   * Deserialized and returns a CompactionDescriptor is the KeyValue contains one.
390   * @param kv the key value
391   * @return deserialized CompactionDescriptor or null.
392   */
393  public static CompactionDescriptor getCompaction(Cell kv) throws IOException {
394    return isCompactionMarker(kv) ? CompactionDescriptor.parseFrom(CellUtil.cloneValue(kv)) : null;
395  }
396
397  /**
398   * Returns true if the given cell is a serialized {@link CompactionDescriptor}
399   * @see #getCompaction(Cell)
400   */
401  public static boolean isCompactionMarker(Cell cell) {
402    return CellUtil.matchingColumn(cell, METAFAMILY, COMPACTION);
403  }
404
405  /**
406   * Create a bulk loader WALEdit
407   * @param hri                The RegionInfo for the region in which we are bulk loading
408   * @param bulkLoadDescriptor The descriptor for the Bulk Loader
409   * @return The WALEdit for the BulkLoad
410   */
411  public static WALEdit createBulkLoadEvent(RegionInfo hri,
412    WALProtos.BulkLoadDescriptor bulkLoadDescriptor) {
413    KeyValue kv = new KeyValue(getRowForRegion(hri), METAFAMILY, BULK_LOAD,
414      EnvironmentEdgeManager.currentTime(), bulkLoadDescriptor.toByteArray());
415    return new WALEdit().add(kv, METAFAMILY);
416  }
417
418  /**
419   * Deserialized and returns a BulkLoadDescriptor from the passed in Cell
420   * @param cell the key value
421   * @return deserialized BulkLoadDescriptor or null.
422   */
423  public static WALProtos.BulkLoadDescriptor getBulkLoadDescriptor(Cell cell) throws IOException {
424    return CellUtil.matchingColumn(cell, METAFAMILY, BULK_LOAD)
425      ? WALProtos.BulkLoadDescriptor.parseFrom(CellUtil.cloneValue(cell))
426      : null;
427  }
428
429  /**
430   * Append the given map of family->edits to a WALEdit data structure. This does not write to the
431   * WAL itself. Note that as an optimization, we will stamp the Set of column families into the
432   * WALEdit to save on our having to calculate column families subsequently down in the actual WAL
433   * writing.
434   * @param familyMap map of family->edits
435   */
436  public void add(Map<byte[], List<Cell>> familyMap) {
437    for (Map.Entry<byte[], List<Cell>> e : familyMap.entrySet()) {
438      // 'foreach' loop NOT used. See HBASE-12023 "...creates too many iterator objects."
439      int listSize = e.getValue().size();
440      // Add all Cells first and then at end, add the family rather than call {@link #add(Cell)}
441      // and have it clone family each time. Optimization!
442      for (int i = 0; i < listSize; i++) {
443        addCell(e.getValue().get(i));
444      }
445      addFamily(e.getKey());
446    }
447  }
448
449  private void addFamily(byte[] family) {
450    getOrCreateFamilies().add(family);
451  }
452
453  private WALEdit addCell(Cell cell) {
454    this.cells.add(cell);
455    return this;
456  }
457}