001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.region;
019
020import static org.apache.hadoop.hbase.HConstants.HREGION_LOGDIR_NAME;
021
022import java.io.IOException;
023import org.apache.hadoop.conf.Configuration;
024import org.apache.hadoop.fs.FileStatus;
025import org.apache.hadoop.fs.FileSystem;
026import org.apache.hadoop.fs.Path;
027import org.apache.hadoop.hbase.HBaseIOException;
028import org.apache.hadoop.hbase.Server;
029import org.apache.hadoop.hbase.TableName;
030import org.apache.hadoop.hbase.client.Get;
031import org.apache.hadoop.hbase.client.RegionInfo;
032import org.apache.hadoop.hbase.client.RegionInfoBuilder;
033import org.apache.hadoop.hbase.client.Result;
034import org.apache.hadoop.hbase.client.Scan;
035import org.apache.hadoop.hbase.client.TableDescriptor;
036import org.apache.hadoop.hbase.regionserver.HRegion;
037import org.apache.hadoop.hbase.regionserver.HRegion.FlushResult;
038import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
039import org.apache.hadoop.hbase.regionserver.RegionScanner;
040import org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL;
041import org.apache.hadoop.hbase.util.Bytes;
042import org.apache.hadoop.hbase.util.CommonFSUtils;
043import org.apache.hadoop.hbase.util.FSUtils;
044import org.apache.hadoop.hbase.util.HFileArchiveUtil;
045import org.apache.hadoop.hbase.util.RecoverLeaseFSUtils;
046import org.apache.hadoop.hbase.wal.AbstractFSWALProvider;
047import org.apache.hadoop.hbase.wal.WAL;
048import org.apache.hadoop.hbase.wal.WALFactory;
049import org.apache.yetus.audience.InterfaceAudience;
050import org.slf4j.Logger;
051import org.slf4j.LoggerFactory;
052
053import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
054import org.apache.hbase.thirdparty.com.google.common.math.IntMath;
055
056/**
057 * A region that stores data in a separated directory, which can be used to store master local data.
058 * <p/>
059 * FileSystem layout:
060 *
061 * <pre>
062 * hbase
063 *   |
064 *   --&lt;region dir&gt;
065 *       |
066 *       --data
067 *       |  |
068 *       |  --/&lt;ns&gt/&lt;table&gt/&lt;encoded-region-name&gt; <---- The region data
069 *       |      |
070 *       |      --replay <---- The edits to replay
071 *       |
072 *       --WALs
073 *          |
074 *          --&lt;master-server-name&gt; <---- The WAL dir for active master
075 *          |
076 *          --&lt;master-server-name&gt;-dead <---- The WAL dir for dead master
077 * </pre>
078 *
079 * Notice that, you can use different root file system and WAL file system. Then the above directory
080 * will be on two file systems, the root file system will have the data directory while the WAL
081 * filesystem will have the WALs directory. The archived HFile will be moved to the global HFile
082 * archived directory with the {@link MasterRegionParams#archivedWalSuffix()} suffix. The archived
083 * WAL will be moved to the global WAL archived directory with the
084 * {@link MasterRegionParams#archivedHFileSuffix()} suffix.
085 */
086@InterfaceAudience.Private
087public final class MasterRegion {
088
089  private static final Logger LOG = LoggerFactory.getLogger(MasterRegion.class);
090
091  private static final String REPLAY_EDITS_DIR = "recovered.wals";
092
093  private static final String DEAD_WAL_DIR_SUFFIX = "-dead";
094
095  private static final int REGION_ID = 1;
096
097  private final WALFactory walFactory;
098
099  @VisibleForTesting
100  final HRegion region;
101
102  @VisibleForTesting
103  final MasterRegionFlusherAndCompactor flusherAndCompactor;
104
105  private MasterRegionWALRoller walRoller;
106
107  private MasterRegion(HRegion region, WALFactory walFactory,
108    MasterRegionFlusherAndCompactor flusherAndCompactor, MasterRegionWALRoller walRoller) {
109    this.region = region;
110    this.walFactory = walFactory;
111    this.flusherAndCompactor = flusherAndCompactor;
112    this.walRoller = walRoller;
113  }
114
115  private void closeRegion(boolean abort) {
116    try {
117      region.close(abort);
118    } catch (IOException e) {
119      LOG.warn("Failed to close region", e);
120    }
121  }
122
123  private void shutdownWAL() {
124    try {
125      walFactory.shutdown();
126    } catch (IOException e) {
127      LOG.warn("Failed to shutdown WAL", e);
128    }
129  }
130
131  public void update(UpdateMasterRegion action) throws IOException {
132    action.update(region);
133    flusherAndCompactor.onUpdate();
134  }
135
136  public Result get(Get get) throws IOException {
137    return region.get(get);
138  }
139
140  public RegionScanner getScanner(Scan scan) throws IOException {
141    return region.getScanner(scan);
142  }
143
144  @VisibleForTesting
145  public FlushResult flush(boolean force) throws IOException {
146    return region.flush(force);
147  }
148
149  @VisibleForTesting
150  public void requestRollAll() {
151    walRoller.requestRollAll();
152  }
153
154  @VisibleForTesting
155  public void waitUntilWalRollFinished() throws InterruptedException {
156    walRoller.waitUntilWalRollFinished();
157  }
158
159  public void close(boolean abort) {
160    LOG.info("Closing local region {}, isAbort={}", region.getRegionInfo(), abort);
161    if (flusherAndCompactor != null) {
162      flusherAndCompactor.close();
163    }
164    // if abort, we shutdown wal first to fail the ongoing updates to the region, and then close the
165    // region, otherwise there will be dead lock.
166    if (abort) {
167      shutdownWAL();
168      closeRegion(true);
169    } else {
170      closeRegion(false);
171      shutdownWAL();
172    }
173
174    if (walRoller != null) {
175      walRoller.close();
176    }
177  }
178
179  private static WAL createWAL(WALFactory walFactory, MasterRegionWALRoller walRoller,
180    String serverName, FileSystem walFs, Path walRootDir, RegionInfo regionInfo)
181    throws IOException {
182    String logName = AbstractFSWALProvider.getWALDirectoryName(serverName);
183    Path walDir = new Path(walRootDir, logName);
184    LOG.debug("WALDir={}", walDir);
185    if (walFs.exists(walDir)) {
186      throw new HBaseIOException(
187        "Already created wal directory at " + walDir + " for local region " + regionInfo);
188    }
189    if (!walFs.mkdirs(walDir)) {
190      throw new IOException(
191        "Can not create wal directory " + walDir + " for local region " + regionInfo);
192    }
193    WAL wal = walFactory.getWAL(regionInfo);
194    walRoller.addWAL(wal);
195    return wal;
196  }
197
198  private static HRegion bootstrap(Configuration conf, TableDescriptor td, FileSystem fs,
199    Path rootDir, FileSystem walFs, Path walRootDir, WALFactory walFactory,
200    MasterRegionWALRoller walRoller, String serverName) throws IOException {
201    TableName tn = td.getTableName();
202    RegionInfo regionInfo = RegionInfoBuilder.newBuilder(tn).setRegionId(REGION_ID).build();
203    Path tmpTableDir = CommonFSUtils.getTableDir(rootDir,
204      TableName.valueOf(tn.getNamespaceAsString(), tn.getQualifierAsString() + "-tmp"));
205    if (fs.exists(tmpTableDir) && !fs.delete(tmpTableDir, true)) {
206      throw new IOException("Can not delete partial created proc region " + tmpTableDir);
207    }
208    HRegion.createHRegion(conf, regionInfo, fs, tmpTableDir, td).close();
209    Path tableDir = CommonFSUtils.getTableDir(rootDir, tn);
210    if (!fs.rename(tmpTableDir, tableDir)) {
211      throw new IOException("Can not rename " + tmpTableDir + " to " + tableDir);
212    }
213    WAL wal = createWAL(walFactory, walRoller, serverName, walFs, walRootDir, regionInfo);
214    return HRegion.openHRegionFromTableDir(conf, fs, tableDir, regionInfo, td, wal, null, null);
215  }
216
217  private static HRegion open(Configuration conf, TableDescriptor td, FileSystem fs, Path rootDir,
218    FileSystem walFs, Path walRootDir, WALFactory walFactory, MasterRegionWALRoller walRoller,
219    String serverName) throws IOException {
220    Path tableDir = CommonFSUtils.getTableDir(rootDir, td.getTableName());
221    Path regionDir =
222      fs.listStatus(tableDir, p -> RegionInfo.isEncodedRegionName(Bytes.toBytes(p.getName())))[0]
223        .getPath();
224    RegionInfo regionInfo = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir);
225
226    Path walRegionDir = FSUtils.getRegionDirFromRootDir(walRootDir, regionInfo);
227    Path replayEditsDir = new Path(walRegionDir, REPLAY_EDITS_DIR);
228    if (!walFs.exists(replayEditsDir) && !walFs.mkdirs(replayEditsDir)) {
229      throw new IOException("Failed to create replay directory: " + replayEditsDir);
230    }
231    Path walsDir = new Path(walRootDir, HREGION_LOGDIR_NAME);
232    for (FileStatus walDir : walFs.listStatus(walsDir)) {
233      if (!walDir.isDirectory()) {
234        continue;
235      }
236      if (walDir.getPath().getName().startsWith(serverName)) {
237        LOG.warn("This should not happen in real production as we have not created our WAL " +
238          "directory yet, ignore if you are running a local region related UT");
239      }
240      Path deadWALDir;
241      if (!walDir.getPath().getName().endsWith(DEAD_WAL_DIR_SUFFIX)) {
242        deadWALDir =
243          new Path(walDir.getPath().getParent(), walDir.getPath().getName() + DEAD_WAL_DIR_SUFFIX);
244        if (!walFs.rename(walDir.getPath(), deadWALDir)) {
245          throw new IOException("Can not rename " + walDir + " to " + deadWALDir +
246            " when recovering lease of proc store");
247        }
248        LOG.info("Renamed {} to {} as it is dead", walDir.getPath(), deadWALDir);
249      } else {
250        deadWALDir = walDir.getPath();
251        LOG.info("{} is already marked as dead", deadWALDir);
252      }
253      for (FileStatus walFile : walFs.listStatus(deadWALDir)) {
254        Path replayEditsFile = new Path(replayEditsDir, walFile.getPath().getName());
255        RecoverLeaseFSUtils.recoverFileLease(walFs, walFile.getPath(), conf);
256        if (!walFs.rename(walFile.getPath(), replayEditsFile)) {
257          throw new IOException("Can not rename " + walFile.getPath() + " to " + replayEditsFile +
258            " when recovering lease for local region");
259        }
260        LOG.info("Renamed {} to {}", walFile.getPath(), replayEditsFile);
261      }
262      LOG.info("Delete empty local region wal dir {}", deadWALDir);
263      walFs.delete(deadWALDir, true);
264    }
265
266    WAL wal = createWAL(walFactory, walRoller, serverName, walFs, walRootDir, regionInfo);
267    conf.set(HRegion.SPECIAL_RECOVERED_EDITS_DIR,
268      replayEditsDir.makeQualified(walFs.getUri(), walFs.getWorkingDirectory()).toString());
269    return HRegion.openHRegionFromTableDir(conf, fs, tableDir, regionInfo, td, wal, null, null);
270  }
271
272  public static MasterRegion create(MasterRegionParams params) throws IOException {
273    TableDescriptor td = params.tableDescriptor();
274    LOG.info("Create or load local region for table " + td);
275    Server server = params.server();
276    Configuration baseConf = server.getConfiguration();
277    FileSystem fs = CommonFSUtils.getRootDirFileSystem(baseConf);
278    FileSystem walFs = CommonFSUtils.getWALFileSystem(baseConf);
279    Path globalRootDir = CommonFSUtils.getRootDir(baseConf);
280    Path globalWALRootDir = CommonFSUtils.getWALRootDir(baseConf);
281    Path rootDir = new Path(globalRootDir, params.regionDirName());
282    Path walRootDir = new Path(globalWALRootDir, params.regionDirName());
283    // we will override some configurations so create a new one.
284    Configuration conf = new Configuration(baseConf);
285    CommonFSUtils.setRootDir(conf, rootDir);
286    CommonFSUtils.setWALRootDir(conf, walRootDir);
287    MasterRegionFlusherAndCompactor.setupConf(conf, params.flushSize(), params.flushPerChanges(),
288      params.flushIntervalMs());
289    conf.setInt(AbstractFSWAL.MAX_LOGS, params.maxWals());
290    if (params.useHsync() != null) {
291      conf.setBoolean(HRegion.WAL_HSYNC_CONF_KEY, params.useHsync());
292    }
293    if (params.useMetaCellComparator() != null) {
294      conf.setBoolean(HRegion.USE_META_CELL_COMPARATOR, params.useMetaCellComparator());
295    }
296    conf.setInt(AbstractFSWAL.RING_BUFFER_SLOT_COUNT,
297      IntMath.ceilingPowerOfTwo(params.ringBufferSlotCount()));
298
299    MasterRegionWALRoller walRoller = MasterRegionWALRoller.create(
300      td.getTableName() + "-WAL-Roller", conf, server, walFs, walRootDir, globalWALRootDir,
301      params.archivedWalSuffix(), params.rollPeriodMs(), params.flushSize());
302    walRoller.start();
303
304    WALFactory walFactory = new WALFactory(conf, server.getServerName().toString());
305    Path tableDir = CommonFSUtils.getTableDir(rootDir, td.getTableName());
306    HRegion region;
307    if (fs.exists(tableDir)) {
308      // load the existing region.
309      region = open(conf, td, fs, rootDir, walFs, walRootDir, walFactory, walRoller,
310        server.getServerName().toString());
311    } else {
312      // bootstrapping...
313      region = bootstrap(conf, td, fs, rootDir, walFs, walRootDir, walFactory, walRoller,
314        server.getServerName().toString());
315    }
316    Path globalArchiveDir = HFileArchiveUtil.getArchivePath(baseConf);
317    MasterRegionFlusherAndCompactor flusherAndCompactor = new MasterRegionFlusherAndCompactor(conf,
318      server, region, params.flushSize(), params.flushPerChanges(), params.flushIntervalMs(),
319      params.compactMin(), globalArchiveDir, params.archivedHFileSuffix());
320    walRoller.setFlusherAndCompactor(flusherAndCompactor);
321    Path archiveDir = HFileArchiveUtil.getArchivePath(conf);
322    if (!fs.mkdirs(archiveDir)) {
323      LOG.warn("Failed to create archive directory {}. Usually this should not happen but it will" +
324        " be created again when we actually archive the hfiles later, so continue", archiveDir);
325    }
326    return new MasterRegion(region, walFactory, flusherAndCompactor, walRoller);
327  }
328}