001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver.wal;
019
020import static org.junit.jupiter.api.Assertions.assertFalse;
021import static org.junit.jupiter.api.Assertions.assertNull;
022import static org.junit.jupiter.api.Assertions.assertTrue;
023
024import java.io.IOException;
025import org.apache.hadoop.conf.Configuration;
026import org.apache.hadoop.fs.FileSystem;
027import org.apache.hadoop.fs.Path;
028import org.apache.hadoop.hbase.HBaseTestingUtil;
029import org.apache.hadoop.hbase.TableName;
030import org.apache.hadoop.hbase.client.Durability;
031import org.apache.hadoop.hbase.client.Put;
032import org.apache.hadoop.hbase.regionserver.ChunkCreator;
033import org.apache.hadoop.hbase.regionserver.HRegion;
034import org.apache.hadoop.hbase.regionserver.MemStoreLAB;
035import org.apache.hadoop.hbase.util.Bytes;
036import org.apache.hadoop.hbase.wal.WAL;
037import org.junit.jupiter.api.AfterEach;
038import org.junit.jupiter.api.BeforeEach;
039import org.junit.jupiter.api.Test;
040import org.junit.jupiter.api.TestInfo;
041
042/**
043 * Tests for WAL write durability - hflush vs hsync
044 */
045public abstract class WALDurabilityTestBase<T extends WAL> {
046
047  private static final String COLUMN_FAMILY = "MyCF";
048  private static final byte[] COLUMN_FAMILY_BYTES = Bytes.toBytes(COLUMN_FAMILY);
049
050  private final static HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
051  private Configuration conf;
052  private String dir;
053  private String name;
054
055  // Test names
056  protected TableName tableName;
057
058  @BeforeEach
059  public void setUp(TestInfo testInfo) throws IOException {
060    name = testInfo.getTestMethod().get().getName();
061    conf = TEST_UTIL.getConfiguration();
062    dir = TEST_UTIL.getDataTestDir("TestHRegion").toString();
063    tableName = TableName.valueOf(name);
064  }
065
066  @AfterEach
067  public void tearDown() throws IOException {
068    TEST_UTIL.cleanupTestDir();
069  }
070
071  protected final T getWAL(FileSystem fs, Path root, String logDir, Configuration conf)
072    throws IOException {
073    fs.mkdirs(new Path(root, logDir));
074    return getWAL0(fs, root, logDir, conf);
075  }
076
077  protected abstract T getWAL0(FileSystem fs, Path root, String logDir, Configuration conf)
078    throws IOException;
079
080  protected abstract void resetSyncFlag(T wal);
081
082  protected abstract Boolean getSyncFlag(T wal);
083
084  protected abstract Boolean getWriterSyncFlag(T wal);
085
086  @Test
087  public void testWALDurability() throws IOException {
088    byte[] bytes = Bytes.toBytes(getName());
089    Put put = new Put(bytes);
090    put.addColumn(COLUMN_FAMILY_BYTES, Bytes.toBytes("1"), bytes);
091
092    // global hbase.wal.hsync false, no override in put call - hflush
093    conf.set(HRegion.WAL_HSYNC_CONF_KEY, "false");
094    FileSystem fs = FileSystem.get(conf);
095    Path rootDir = new Path(dir + getName());
096    T wal = getWAL(fs, rootDir, getName(), conf);
097    HRegion region = initHRegion(tableName, null, null, conf, wal);
098    try {
099      resetSyncFlag(wal);
100      assertNull(getSyncFlag(wal));
101      assertNull(getWriterSyncFlag(wal));
102      region.put(put);
103      assertFalse(getSyncFlag(wal));
104      assertFalse(getWriterSyncFlag(wal));
105
106      // global hbase.wal.hsync false, durability set in put call - fsync
107      put.setDurability(Durability.FSYNC_WAL);
108      resetSyncFlag(wal);
109      assertNull(getSyncFlag(wal));
110      assertNull(getWriterSyncFlag(wal));
111      region.put(put);
112      assertTrue(getSyncFlag(wal));
113      assertTrue(getWriterSyncFlag(wal));
114    } finally {
115      HBaseTestingUtil.closeRegionAndWAL(region);
116    }
117
118    // global hbase.wal.hsync true, no override in put call
119    conf.set(HRegion.WAL_HSYNC_CONF_KEY, "true");
120    fs = FileSystem.get(conf);
121    wal = getWAL(fs, rootDir, getName(), conf);
122    region = initHRegion(tableName, null, null, conf, wal);
123
124    try {
125      resetSyncFlag(wal);
126      assertNull(getSyncFlag(wal));
127      assertNull(getWriterSyncFlag(wal));
128      region.put(put);
129      assertTrue(getSyncFlag(wal));
130      assertTrue(getWriterSyncFlag(wal));
131
132      // global hbase.wal.hsync true, durability set in put call - fsync
133      put.setDurability(Durability.FSYNC_WAL);
134      resetSyncFlag(wal);
135      assertNull(getSyncFlag(wal));
136      assertNull(getWriterSyncFlag(wal));
137      region.put(put);
138      assertTrue(getSyncFlag(wal));
139      assertTrue(getWriterSyncFlag(wal));
140
141      // global hbase.wal.hsync true, durability set in put call - sync
142      put = new Put(bytes);
143      put.addColumn(COLUMN_FAMILY_BYTES, Bytes.toBytes("1"), bytes);
144      put.setDurability(Durability.SYNC_WAL);
145      resetSyncFlag(wal);
146      assertNull(getSyncFlag(wal));
147      assertNull(getWriterSyncFlag(wal));
148      region.put(put);
149      assertFalse(getSyncFlag(wal));
150      assertFalse(getWriterSyncFlag(wal));
151    } finally {
152      HBaseTestingUtil.closeRegionAndWAL(region);
153    }
154  }
155
156  private String getName() {
157    return name;
158  }
159
160  /**
161   * @return A region on which you must call {@link HBaseTestingUtil#closeRegionAndWAL(HRegion)}
162   *         when done.
163   */
164  public static HRegion initHRegion(TableName tableName, byte[] startKey, byte[] stopKey,
165    Configuration conf, WAL wal) throws IOException {
166    ChunkCreator.initialize(MemStoreLAB.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null,
167      MemStoreLAB.INDEX_CHUNK_SIZE_PERCENTAGE_DEFAULT);
168    return TEST_UTIL.createLocalHRegion(tableName, startKey, stopKey, conf, false,
169      Durability.USE_DEFAULT, wal, COLUMN_FAMILY_BYTES);
170  }
171}