001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019package org.apache.hadoop.hbase.regionserver.wal;
020
021import java.io.IOException;
022import java.util.NavigableMap;
023
024import org.apache.hadoop.conf.Configuration;
025import org.apache.hadoop.fs.FileSystem;
026import org.apache.hadoop.fs.Path;
027import org.apache.hadoop.hbase.client.RegionInfo;
028import org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl;
029import org.apache.hadoop.hbase.util.CommonFSUtils;
030import org.apache.hadoop.hbase.wal.WAL;
031import org.apache.hadoop.hbase.wal.WALEdit;
032import org.apache.hadoop.hbase.wal.WALKeyImpl;
033import org.apache.yetus.audience.InterfaceAudience;
034import org.slf4j.Logger;
035import org.slf4j.LoggerFactory;
036import org.apache.hbase.thirdparty.com.google.protobuf.TextFormat;
037import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos;
038import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.CompactionDescriptor;
039import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.FlushDescriptor;
040import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.RegionEventDescriptor;
041
042/**
043 * Helper methods to ease Region Server integration with the Write Ahead Log (WAL).
044 * Note that methods in this class specifically should not require access to anything
045 * other than the API found in {@link WAL}. For internal use only.
046 */
047@InterfaceAudience.Private
048public class WALUtil {
049  private static final Logger LOG = LoggerFactory.getLogger(WALUtil.class);
050
051  private WALUtil() {
052    // Shut down construction of this class.
053  }
054
055  /**
056   * Write the marker that a compaction has succeeded and is about to be committed. This provides
057   * info to the HMaster to allow it to recover the compaction if this regionserver dies in the
058   * middle. It also prevents the compaction from finishing if this regionserver has already lost
059   * its lease on the log.
060   * <p/>
061   * This write is for internal use only. Not for external client consumption.
062   * @param mvcc Used by WAL to get sequence Id for the waledit.
063   */
064  public static WALKeyImpl writeCompactionMarker(WAL wal,
065    NavigableMap<byte[], Integer> replicationScope, RegionInfo hri, final CompactionDescriptor c,
066    MultiVersionConcurrencyControl mvcc) throws IOException {
067    WALKeyImpl walKey =
068      writeMarker(wal, replicationScope, hri, WALEdit.createCompaction(hri, c), mvcc);
069    if (LOG.isTraceEnabled()) {
070      LOG.trace("Appended compaction marker " + TextFormat.shortDebugString(c));
071    }
072    return walKey;
073  }
074
075  /**
076   * Write a flush marker indicating a start / abort or a complete of a region flush
077   * <p/>
078   * This write is for internal use only. Not for external client consumption.
079   */
080  public static WALKeyImpl writeFlushMarker(WAL wal, NavigableMap<byte[], Integer> replicationScope,
081    RegionInfo hri, final FlushDescriptor f, boolean sync, MultiVersionConcurrencyControl mvcc)
082    throws IOException {
083    WALKeyImpl walKey = doFullMarkerAppendTransaction(wal, replicationScope, hri,
084      WALEdit.createFlushWALEdit(hri, f), mvcc, sync);
085    if (LOG.isTraceEnabled()) {
086      LOG.trace("Appended flush marker " + TextFormat.shortDebugString(f));
087    }
088    return walKey;
089  }
090
091  /**
092   * Write a region open marker indicating that the region is opened. This write is for internal use
093   * only. Not for external client consumption.
094   */
095  public static WALKeyImpl writeRegionEventMarker(WAL wal,
096      NavigableMap<byte[], Integer> replicationScope, RegionInfo hri, RegionEventDescriptor r,
097      MultiVersionConcurrencyControl mvcc)
098    throws IOException {
099    WALKeyImpl walKey =
100        writeMarker(wal, replicationScope, hri, WALEdit.createRegionEventWALEdit(hri, r), mvcc);
101    if (LOG.isTraceEnabled()) {
102      LOG.trace("Appended region event marker " + TextFormat.shortDebugString(r));
103    }
104    return walKey;
105  }
106
107  /**
108   * Write a log marker that a bulk load has succeeded and is about to be committed.
109   * This write is for internal use only. Not for external client consumption.
110   * @param wal The log to write into.
111   * @param replicationScope The replication scope of the families in the HRegion
112   * @param hri A description of the region in the table that we are bulk loading into.
113   * @param desc A protocol buffers based description of the client's bulk loading request
114   * @return walKey with sequenceid filled out for this bulk load marker
115   * @throws IOException We will throw an IOException if we can not append to the HLog.
116   */
117  public static WALKeyImpl writeBulkLoadMarkerAndSync(final WAL wal,
118      final NavigableMap<byte[], Integer> replicationScope, final RegionInfo hri,
119      final WALProtos.BulkLoadDescriptor desc, final MultiVersionConcurrencyControl mvcc)
120    throws IOException {
121    WALKeyImpl walKey = writeMarker(wal, replicationScope, hri,
122      WALEdit.createBulkLoadEvent(hri, desc), mvcc);
123    if (LOG.isTraceEnabled()) {
124      LOG.trace("Appended Bulk Load marker " + TextFormat.shortDebugString(desc));
125    }
126    return walKey;
127  }
128
129  private static WALKeyImpl writeMarker(final WAL wal, NavigableMap<byte[],
130      Integer> replicationScope, RegionInfo hri, WALEdit edit, MultiVersionConcurrencyControl mvcc)
131    throws IOException {
132    // If sync == true in below, then timeout is not used; safe to pass UNSPECIFIED_TIMEOUT
133    return doFullMarkerAppendTransaction(wal, replicationScope, hri, edit, mvcc, true);
134  }
135
136  /**
137   * A 'full' WAL transaction involves starting an mvcc transaction followed by an append, an
138   * optional sync, and then a call to complete the mvcc transaction. This method does it all. Good
139   * for case of adding a single edit or marker to the WAL.
140   * <p/>
141   * This write is for internal use only. Not for external client consumption.
142   * @return WALKeyImpl that was added to the WAL.
143   */
144  private static WALKeyImpl doFullMarkerAppendTransaction(final WAL wal,
145    final NavigableMap<byte[], Integer> replicationScope, final RegionInfo hri, final WALEdit edit,
146    final MultiVersionConcurrencyControl mvcc, final boolean sync)
147    throws IOException {
148    // TODO: Pass in current time to use?
149    WALKeyImpl walKey = new WALKeyImpl(hri.getEncodedNameAsBytes(), hri.getTable(),
150      System.currentTimeMillis(), mvcc, replicationScope);
151    long trx = MultiVersionConcurrencyControl.NONE;
152    try {
153      trx = wal.appendMarker(hri, walKey, edit);
154      if (sync) {
155        wal.sync(trx);
156      }
157      // Call complete only here because these are markers only. They are not for clients to read.
158      mvcc.complete(walKey.getWriteEntry());
159    } catch (IOException ioe) {
160      if (walKey.getWriteEntry() != null) {
161        mvcc.complete(walKey.getWriteEntry());
162      }
163      throw ioe;
164    }
165    return walKey;
166  }
167
168  /**
169   * Blocksize returned here is 2x the default HDFS blocksize unless explicitly set in
170   * Configuration. Works in tandem with hbase.regionserver.logroll.multiplier. See comment in
171   * AbstractFSWAL in Constructor where we set blocksize and logrollsize for why.
172   * @return Blocksize to use writing WALs.
173   */
174  public static long getWALBlockSize(Configuration conf, FileSystem fs, Path dir)
175      throws IOException {
176    return getWALBlockSize(conf, fs, dir, false);
177  }
178
179  /**
180   * Public because of FSHLog. Should be package-private
181   * @param isRecoverEdits the created writer is for recovered edits or WAL. For recovered edits, it
182   *          is true and for WAL it is false.
183   */
184  public static long getWALBlockSize(Configuration conf, FileSystem fs, Path dir,
185      boolean isRecoverEdits) throws IOException {
186    long defaultBlockSize = CommonFSUtils.getDefaultBlockSize(fs, dir) * 2;
187    if (isRecoverEdits) {
188      return conf.getLong("hbase.regionserver.recoverededits.blocksize", defaultBlockSize);
189    }
190    return conf.getLong("hbase.regionserver.hlog.blocksize", defaultBlockSize);
191  }
192}