001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.wal;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertNotNull;
022import static org.junit.Assert.assertNotSame;
023import static org.junit.Assert.assertNull;
024import static org.junit.Assert.assertTrue;
025
026import java.io.IOException;
027import java.util.NavigableSet;
028import org.apache.hadoop.conf.Configuration;
029import org.apache.hadoop.fs.FSDataOutputStream;
030import org.apache.hadoop.fs.FileSystem;
031import org.apache.hadoop.fs.Path;
032import org.apache.hadoop.hbase.HBaseClassTestRule;
033import org.apache.hadoop.hbase.HBaseTestingUtility;
034import org.apache.hadoop.hbase.HConstants;
035import org.apache.hadoop.hbase.KeyValueTestUtil;
036import org.apache.hadoop.hbase.ServerName;
037import org.apache.hadoop.hbase.TableName;
038import org.apache.hadoop.hbase.testclassification.RegionServerTests;
039import org.apache.hadoop.hbase.testclassification.SmallTests;
040import org.apache.hadoop.hbase.util.Bytes;
041import org.apache.hadoop.hbase.util.CommonFSUtils;
042import org.apache.hadoop.hbase.wal.WALSplitter.PipelineController;
043import org.junit.ClassRule;
044import org.junit.Test;
045import org.junit.experimental.categories.Category;
046
047/**
048 * Simple testing of a few WAL methods.
049 */
050@Category({RegionServerTests.class, SmallTests.class})
051public class TestWALMethods {
052
053  @ClassRule
054  public static final HBaseClassTestRule CLASS_RULE =
055      HBaseClassTestRule.forClass(TestWALMethods.class);
056
057  private static final byte[] TEST_REGION = Bytes.toBytes("test_region");
058  private static final TableName TEST_TABLE =
059      TableName.valueOf("test_table");
060
061  private final HBaseTestingUtility util = new HBaseTestingUtility();
062
063  @Test
064  public void testServerNameFromWAL() throws Exception {
065    Path walPath = new Path("/hbase/WALs/regionserver-2.example.com,22101,1487767381290",
066        "regionserver-2.example.com%2C22101%2C1487767381290.null0.1487785392316");
067    ServerName name = AbstractFSWALProvider.getServerNameFromWALDirectoryName(walPath);
068    assertEquals(ServerName.valueOf("regionserver-2.example.com", 22101, 1487767381290L), name);
069  }
070
071  @Test
072  public void testServerNameFromTestWAL() throws Exception {
073    Path walPath = new Path(
074        "/user/example/test-data/12ff1404-68c6-4715-a4b9-775e763842bc/WALs/TestWALRecordReader",
075        "TestWALRecordReader.default.1487787939118");
076    ServerName name = AbstractFSWALProvider.getServerNameFromWALDirectoryName(walPath);
077    assertNull(name);
078  }
079
080  /**
081   * Assert that getSplitEditFilesSorted returns files in expected order and
082   * that it skips moved-aside files.
083   * @throws IOException
084   */
085  @Test public void testGetSplitEditFilesSorted() throws IOException {
086    FileSystem fs = FileSystem.get(util.getConfiguration());
087    Path regiondir = util.getDataTestDir("regiondir");
088    fs.delete(regiondir, true);
089    fs.mkdirs(regiondir);
090    Path recoverededits = WALSplitUtil.getRegionDirRecoveredEditsDir(regiondir);
091    String first = WALSplitUtil.formatRecoveredEditsFileName(-1);
092    createFile(fs, recoverededits, first);
093    createFile(fs, recoverededits, WALSplitUtil.formatRecoveredEditsFileName(0));
094    createFile(fs, recoverededits, WALSplitUtil.formatRecoveredEditsFileName(1));
095    createFile(fs, recoverededits, WALSplitUtil
096        .formatRecoveredEditsFileName(11));
097    createFile(fs, recoverededits, WALSplitUtil.formatRecoveredEditsFileName(2));
098    createFile(fs, recoverededits, WALSplitUtil
099        .formatRecoveredEditsFileName(50));
100    String last = WALSplitUtil.formatRecoveredEditsFileName(Long.MAX_VALUE);
101    createFile(fs, recoverededits, last);
102    createFile(fs, recoverededits,
103      Long.toString(Long.MAX_VALUE) + "." + System.currentTimeMillis());
104
105    final Configuration walConf = new Configuration(util.getConfiguration());
106    CommonFSUtils.setRootDir(walConf, regiondir);
107    (new WALFactory(walConf, "dummyLogName")).getWAL(null);
108
109    NavigableSet<Path> files = WALSplitUtil.getSplitEditFilesSorted(fs, regiondir);
110    assertEquals(7, files.size());
111    assertEquals(files.pollFirst().getName(), first);
112    assertEquals(files.pollLast().getName(), last);
113    assertEquals(files.pollFirst().getName(),
114        WALSplitUtil
115        .formatRecoveredEditsFileName(0));
116    assertEquals(files.pollFirst().getName(),
117        WALSplitUtil
118        .formatRecoveredEditsFileName(1));
119    assertEquals(files.pollFirst().getName(),
120        WALSplitUtil
121        .formatRecoveredEditsFileName(2));
122    assertEquals(files.pollFirst().getName(),
123        WALSplitUtil
124        .formatRecoveredEditsFileName(11));
125  }
126
127  private void createFile(final FileSystem fs, final Path testdir,
128      final String name)
129  throws IOException {
130    FSDataOutputStream fdos = fs.create(new Path(testdir, name), true);
131    fdos.close();
132  }
133
134  @Test
135  public void testRegionEntryBuffer() throws Exception {
136    EntryBuffers.RegionEntryBuffer reb = new EntryBuffers.RegionEntryBuffer(
137        TEST_TABLE, TEST_REGION);
138    assertEquals(0, reb.heapSize());
139
140    reb.appendEntry(createTestLogEntry(1));
141    assertTrue(reb.heapSize() > 0);
142  }
143
144  @Test
145  public void testEntrySink() throws Exception {
146    EntryBuffers sink = new EntryBuffers(new PipelineController(), 1*1024*1024);
147    for (int i = 0; i < 1000; i++) {
148      WAL.Entry entry = createTestLogEntry(i);
149      sink.appendEntry(entry);
150    }
151
152    assertTrue(sink.totalBuffered > 0);
153    long amountInChunk = sink.totalBuffered;
154    // Get a chunk
155    EntryBuffers.RegionEntryBuffer chunk = sink.getChunkToWrite();
156    assertEquals(chunk.heapSize(), amountInChunk);
157
158    // Make sure it got marked that a thread is "working on this"
159    assertTrue(sink.isRegionCurrentlyWriting(TEST_REGION));
160
161    // Insert some more entries
162    for (int i = 0; i < 500; i++) {
163      WAL.Entry entry = createTestLogEntry(i);
164      sink.appendEntry(entry);
165    }
166    // Asking for another chunk shouldn't work since the first one
167    // is still writing
168    assertNull(sink.getChunkToWrite());
169
170    // If we say we're done writing the first chunk, then we should be able
171    // to get the second
172    sink.doneWriting(chunk);
173
174    EntryBuffers.RegionEntryBuffer chunk2 = sink.getChunkToWrite();
175    assertNotNull(chunk2);
176    assertNotSame(chunk, chunk2);
177    long amountInChunk2 = sink.totalBuffered;
178    // The second chunk had fewer rows than the first
179    assertTrue(amountInChunk2 < amountInChunk);
180
181    sink.doneWriting(chunk2);
182    assertEquals(0, sink.totalBuffered);
183  }
184
185  private WAL.Entry createTestLogEntry(int i) {
186    long seq = i;
187    long now = i * 1000;
188
189    WALEdit edit = new WALEdit();
190    edit.add(KeyValueTestUtil.create("row", "fam", "qual", 1234, "val"));
191    WALKeyImpl key = new WALKeyImpl(TEST_REGION, TEST_TABLE, seq, now,
192        HConstants.DEFAULT_CLUSTER_ID);
193    WAL.Entry entry = new WAL.Entry(key, edit);
194    return entry;
195  }
196
197}
198