001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.wal;
019
020import static org.junit.jupiter.api.Assertions.assertEquals;
021
022import java.io.IOException;
023import java.util.ArrayList;
024import java.util.Collections;
025import java.util.List;
026import org.apache.hadoop.conf.Configuration;
027import org.apache.hadoop.fs.FileStatus;
028import org.apache.hadoop.fs.FileSystem;
029import org.apache.hadoop.fs.Path;
030import org.apache.hadoop.hbase.HBaseTestingUtil;
031import org.apache.hadoop.hbase.HConstants;
032import org.apache.hadoop.hbase.KeyValue;
033import org.apache.hadoop.hbase.TableName;
034import org.apache.hadoop.hbase.client.RegionInfo;
035import org.apache.hadoop.hbase.client.RegionInfoBuilder;
036import org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl;
037import org.apache.hadoop.hbase.testclassification.MediumTests;
038import org.apache.hadoop.hbase.util.Bytes;
039import org.apache.hadoop.hbase.util.CommonFSUtils;
040import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
041import org.junit.jupiter.api.AfterAll;
042import org.junit.jupiter.api.BeforeAll;
043import org.junit.jupiter.api.BeforeEach;
044import org.junit.jupiter.api.Tag;
045import org.junit.jupiter.api.Test;
046import org.slf4j.Logger;
047import org.slf4j.LoggerFactory;
048
049@Tag(MediumTests.TAG)
050public class TestWALRootDir {
051
052  private static final Logger LOG = LoggerFactory.getLogger(TestWALRootDir.class);
053  private final static HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
054  private static Configuration conf;
055  private static FileSystem fs;
056  private static FileSystem walFs;
057  private static final TableName tableName = TableName.valueOf("TestWALWALDir");
058  private static final byte[] rowName = Bytes.toBytes("row");
059  private static final byte[] family = Bytes.toBytes("column");
060  private static Path walRootDir;
061  private static Path rootDir;
062  private static WALFactory wals;
063
064  @BeforeEach
065  public void setUp() throws Exception {
066    cleanup();
067  }
068
069  @BeforeAll
070  public static void setUpBeforeClass() throws Exception {
071    conf = TEST_UTIL.getConfiguration();
072    TEST_UTIL.startMiniDFSCluster(1);
073    rootDir = TEST_UTIL.createRootDir();
074    walRootDir = TEST_UTIL.createWALRootDir();
075    fs = CommonFSUtils.getRootDirFileSystem(conf);
076    walFs = CommonFSUtils.getWALFileSystem(conf);
077  }
078
079  @AfterAll
080  public static void tearDownAfterClass() throws Exception {
081    cleanup();
082    TEST_UTIL.shutdownMiniDFSCluster();
083  }
084
085  @Test
086  public void testWALRootDir() throws Exception {
087    RegionInfo regionInfo = RegionInfoBuilder.newBuilder(tableName).build();
088    wals = new WALFactory(conf, "testWALRootDir");
089    WAL log = wals.getWAL(regionInfo);
090
091    assertEquals(1, getWALFiles(walFs, walRootDir).size());
092    byte[] value = Bytes.toBytes("value");
093    WALEdit edit = new WALEdit();
094    edit.add(new KeyValue(rowName, family, Bytes.toBytes("1"), EnvironmentEdgeManager.currentTime(),
095      value));
096    long txid = log.appendData(regionInfo,
097      getWalKey(EnvironmentEdgeManager.currentTime(), regionInfo, 0), edit);
098    log.sync(txid);
099    assertEquals(1, getWALFiles(walFs, walRootDir).size(), "Expect 1 log have been created");
100    log.rollWriter();
101    // Create 1 more WAL
102    assertEquals(2,
103      getWALFiles(walFs, new Path(walRootDir, HConstants.HREGION_LOGDIR_NAME)).size());
104    edit.add(new KeyValue(rowName, family, Bytes.toBytes("2"), EnvironmentEdgeManager.currentTime(),
105      value));
106    txid = log.appendData(regionInfo,
107      getWalKey(EnvironmentEdgeManager.currentTime(), regionInfo, 1), edit);
108    log.sync(txid);
109    log.rollWriter();
110    log.shutdown();
111
112    assertEquals(3, getWALFiles(walFs, new Path(walRootDir, HConstants.HREGION_LOGDIR_NAME)).size(),
113      "Expect 3 logs in WALs dir");
114  }
115
116  private WALKeyImpl getWalKey(final long time, RegionInfo hri, final long startPoint) {
117    return new WALKeyImpl(hri.getEncodedNameAsBytes(), tableName, time,
118      new MultiVersionConcurrencyControl(startPoint));
119  }
120
121  private List<FileStatus> getWALFiles(FileSystem fs, Path dir) throws IOException {
122    List<FileStatus> result = new ArrayList<FileStatus>();
123    LOG.debug("Scanning " + dir.toString() + " for WAL files");
124
125    FileStatus[] files = fs.listStatus(dir);
126    if (files == null) return Collections.emptyList();
127    for (FileStatus file : files) {
128      if (file.isDirectory()) {
129        // recurse into sub directories
130        result.addAll(getWALFiles(fs, file.getPath()));
131      } else {
132        String name = file.getPath().toString();
133        if (!name.startsWith(".")) {
134          result.add(file);
135        }
136      }
137    }
138    return result;
139  }
140
141  private static void cleanup() throws Exception {
142    walFs.delete(walRootDir, true);
143    fs.delete(rootDir, true);
144  }
145
146}