001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.region;
019
020import static org.apache.hadoop.hbase.HConstants.HREGION_LOGDIR_NAME;
021
022import java.io.IOException;
023import org.apache.hadoop.conf.Configuration;
024import org.apache.hadoop.fs.FileStatus;
025import org.apache.hadoop.fs.FileSystem;
026import org.apache.hadoop.fs.Path;
027import org.apache.hadoop.hbase.HBaseIOException;
028import org.apache.hadoop.hbase.Server;
029import org.apache.hadoop.hbase.TableName;
030import org.apache.hadoop.hbase.client.Get;
031import org.apache.hadoop.hbase.client.RegionInfo;
032import org.apache.hadoop.hbase.client.RegionInfoBuilder;
033import org.apache.hadoop.hbase.client.Result;
034import org.apache.hadoop.hbase.client.Scan;
035import org.apache.hadoop.hbase.client.TableDescriptor;
036import org.apache.hadoop.hbase.regionserver.HRegion;
037import org.apache.hadoop.hbase.regionserver.HRegion.FlushResult;
038import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
039import org.apache.hadoop.hbase.regionserver.RegionScanner;
040import org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL;
041import org.apache.hadoop.hbase.util.Bytes;
042import org.apache.hadoop.hbase.util.CommonFSUtils;
043import org.apache.hadoop.hbase.util.FSUtils;
044import org.apache.hadoop.hbase.util.HFileArchiveUtil;
045import org.apache.hadoop.hbase.util.RecoverLeaseFSUtils;
046import org.apache.hadoop.hbase.wal.AbstractFSWALProvider;
047import org.apache.hadoop.hbase.wal.WAL;
048import org.apache.hadoop.hbase.wal.WALFactory;
049import org.apache.yetus.audience.InterfaceAudience;
050import org.slf4j.Logger;
051import org.slf4j.LoggerFactory;
052
053import org.apache.hbase.thirdparty.com.google.common.math.IntMath;
054
055/**
056 * A region that stores data in a separated directory, which can be used to store master local data.
057 * <p/>
058 * FileSystem layout:
059 *
060 * <pre>
061 * hbase
062 *   |
063 *   --&lt;region dir&gt;
064 *       |
065 *       --data
066 *       |  |
067 *       |  --/&lt;ns&gt/&lt;table&gt/&lt;encoded-region-name&gt; <---- The region data
068 *       |      |
069 *       |      --replay <---- The edits to replay
070 *       |
071 *       --WALs
072 *          |
073 *          --&lt;master-server-name&gt; <---- The WAL dir for active master
074 *          |
075 *          --&lt;master-server-name&gt;-dead <---- The WAL dir for dead master
076 * </pre>
077 *
078 * Notice that, you can use different root file system and WAL file system. Then the above directory
079 * will be on two file systems, the root file system will have the data directory while the WAL
080 * filesystem will have the WALs directory. The archived HFile will be moved to the global HFile
081 * archived directory with the {@link MasterRegionParams#archivedWalSuffix()} suffix. The archived
082 * WAL will be moved to the global WAL archived directory with the
083 * {@link MasterRegionParams#archivedHFileSuffix()} suffix.
084 */
085@InterfaceAudience.Private
086public final class MasterRegion {
087
088  private static final Logger LOG = LoggerFactory.getLogger(MasterRegion.class);
089
090  private static final String REPLAY_EDITS_DIR = "recovered.wals";
091
092  private static final String DEAD_WAL_DIR_SUFFIX = "-dead";
093
094  private static final int REGION_ID = 1;
095
096  private final WALFactory walFactory;
097
098  final HRegion region;
099
100  final MasterRegionFlusherAndCompactor flusherAndCompactor;
101
102  private MasterRegionWALRoller walRoller;
103
104  private MasterRegion(HRegion region, WALFactory walFactory,
105    MasterRegionFlusherAndCompactor flusherAndCompactor, MasterRegionWALRoller walRoller) {
106    this.region = region;
107    this.walFactory = walFactory;
108    this.flusherAndCompactor = flusherAndCompactor;
109    this.walRoller = walRoller;
110  }
111
112  private void closeRegion(boolean abort) {
113    try {
114      region.close(abort);
115    } catch (IOException e) {
116      LOG.warn("Failed to close region", e);
117    }
118  }
119
120  private void shutdownWAL() {
121    try {
122      walFactory.shutdown();
123    } catch (IOException e) {
124      LOG.warn("Failed to shutdown WAL", e);
125    }
126  }
127
128  public void update(UpdateMasterRegion action) throws IOException {
129    action.update(region);
130    flusherAndCompactor.onUpdate();
131  }
132
133  public Result get(Get get) throws IOException {
134    return region.get(get);
135  }
136
137  public RegionScanner getScanner(Scan scan) throws IOException {
138    return region.getScanner(scan);
139  }
140
141  public FlushResult flush(boolean force) throws IOException {
142    return region.flush(force);
143  }
144
145  public void requestRollAll() {
146    walRoller.requestRollAll();
147  }
148
149  public void waitUntilWalRollFinished() throws InterruptedException {
150    walRoller.waitUntilWalRollFinished();
151  }
152
153  public void close(boolean abort) {
154    LOG.info("Closing local region {}, isAbort={}", region.getRegionInfo(), abort);
155    if (flusherAndCompactor != null) {
156      flusherAndCompactor.close();
157    }
158    // if abort, we shutdown wal first to fail the ongoing updates to the region, and then close the
159    // region, otherwise there will be dead lock.
160    if (abort) {
161      shutdownWAL();
162      closeRegion(true);
163    } else {
164      closeRegion(false);
165      shutdownWAL();
166    }
167
168    if (walRoller != null) {
169      walRoller.close();
170    }
171  }
172
173  private static WAL createWAL(WALFactory walFactory, MasterRegionWALRoller walRoller,
174    String serverName, FileSystem walFs, Path walRootDir, RegionInfo regionInfo)
175    throws IOException {
176    String logName = AbstractFSWALProvider.getWALDirectoryName(serverName);
177    Path walDir = new Path(walRootDir, logName);
178    LOG.debug("WALDir={}", walDir);
179    if (walFs.exists(walDir)) {
180      throw new HBaseIOException(
181        "Already created wal directory at " + walDir + " for local region " + regionInfo);
182    }
183    if (!walFs.mkdirs(walDir)) {
184      throw new IOException(
185        "Can not create wal directory " + walDir + " for local region " + regionInfo);
186    }
187    WAL wal = walFactory.getWAL(regionInfo);
188    walRoller.addWAL(wal);
189    return wal;
190  }
191
192  private static HRegion bootstrap(Configuration conf, TableDescriptor td, FileSystem fs,
193    Path rootDir, FileSystem walFs, Path walRootDir, WALFactory walFactory,
194    MasterRegionWALRoller walRoller, String serverName) throws IOException {
195    TableName tn = td.getTableName();
196    RegionInfo regionInfo = RegionInfoBuilder.newBuilder(tn).setRegionId(REGION_ID).build();
197    Path tmpTableDir = CommonFSUtils.getTableDir(rootDir,
198      TableName.valueOf(tn.getNamespaceAsString(), tn.getQualifierAsString() + "-tmp"));
199    if (fs.exists(tmpTableDir) && !fs.delete(tmpTableDir, true)) {
200      throw new IOException("Can not delete partial created proc region " + tmpTableDir);
201    }
202    HRegion.createHRegion(conf, regionInfo, fs, tmpTableDir, td).close();
203    Path tableDir = CommonFSUtils.getTableDir(rootDir, tn);
204    if (!fs.rename(tmpTableDir, tableDir)) {
205      throw new IOException("Can not rename " + tmpTableDir + " to " + tableDir);
206    }
207    WAL wal = createWAL(walFactory, walRoller, serverName, walFs, walRootDir, regionInfo);
208    return HRegion.openHRegionFromTableDir(conf, fs, tableDir, regionInfo, td, wal, null, null);
209  }
210
211  private static HRegion open(Configuration conf, TableDescriptor td, FileSystem fs, Path rootDir,
212    FileSystem walFs, Path walRootDir, WALFactory walFactory, MasterRegionWALRoller walRoller,
213    String serverName) throws IOException {
214    Path tableDir = CommonFSUtils.getTableDir(rootDir, td.getTableName());
215    Path regionDir =
216      fs.listStatus(tableDir, p -> RegionInfo.isEncodedRegionName(Bytes.toBytes(p.getName())))[0]
217        .getPath();
218    RegionInfo regionInfo = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir);
219
220    Path walRegionDir = FSUtils.getRegionDirFromRootDir(walRootDir, regionInfo);
221    Path replayEditsDir = new Path(walRegionDir, REPLAY_EDITS_DIR);
222    if (!walFs.exists(replayEditsDir) && !walFs.mkdirs(replayEditsDir)) {
223      throw new IOException("Failed to create replay directory: " + replayEditsDir);
224    }
225    Path walsDir = new Path(walRootDir, HREGION_LOGDIR_NAME);
226    for (FileStatus walDir : walFs.listStatus(walsDir)) {
227      if (!walDir.isDirectory()) {
228        continue;
229      }
230      if (walDir.getPath().getName().startsWith(serverName)) {
231        LOG.warn("This should not happen in real production as we have not created our WAL " +
232          "directory yet, ignore if you are running a local region related UT");
233      }
234      Path deadWALDir;
235      if (!walDir.getPath().getName().endsWith(DEAD_WAL_DIR_SUFFIX)) {
236        deadWALDir =
237          new Path(walDir.getPath().getParent(), walDir.getPath().getName() + DEAD_WAL_DIR_SUFFIX);
238        if (!walFs.rename(walDir.getPath(), deadWALDir)) {
239          throw new IOException("Can not rename " + walDir + " to " + deadWALDir +
240            " when recovering lease of proc store");
241        }
242        LOG.info("Renamed {} to {} as it is dead", walDir.getPath(), deadWALDir);
243      } else {
244        deadWALDir = walDir.getPath();
245        LOG.info("{} is already marked as dead", deadWALDir);
246      }
247      for (FileStatus walFile : walFs.listStatus(deadWALDir)) {
248        Path replayEditsFile = new Path(replayEditsDir, walFile.getPath().getName());
249        RecoverLeaseFSUtils.recoverFileLease(walFs, walFile.getPath(), conf);
250        if (!walFs.rename(walFile.getPath(), replayEditsFile)) {
251          throw new IOException("Can not rename " + walFile.getPath() + " to " + replayEditsFile +
252            " when recovering lease for local region");
253        }
254        LOG.info("Renamed {} to {}", walFile.getPath(), replayEditsFile);
255      }
256      LOG.info("Delete empty local region wal dir {}", deadWALDir);
257      walFs.delete(deadWALDir, true);
258    }
259
260    WAL wal = createWAL(walFactory, walRoller, serverName, walFs, walRootDir, regionInfo);
261    conf.set(HRegion.SPECIAL_RECOVERED_EDITS_DIR,
262      replayEditsDir.makeQualified(walFs.getUri(), walFs.getWorkingDirectory()).toString());
263    return HRegion.openHRegionFromTableDir(conf, fs, tableDir, regionInfo, td, wal, null, null);
264  }
265
266  public static MasterRegion create(MasterRegionParams params) throws IOException {
267    TableDescriptor td = params.tableDescriptor();
268    LOG.info("Create or load local region for table " + td);
269    Server server = params.server();
270    Configuration baseConf = server.getConfiguration();
271    FileSystem fs = CommonFSUtils.getRootDirFileSystem(baseConf);
272    FileSystem walFs = CommonFSUtils.getWALFileSystem(baseConf);
273    Path globalRootDir = CommonFSUtils.getRootDir(baseConf);
274    Path globalWALRootDir = CommonFSUtils.getWALRootDir(baseConf);
275    Path rootDir = new Path(globalRootDir, params.regionDirName());
276    Path walRootDir = new Path(globalWALRootDir, params.regionDirName());
277    // we will override some configurations so create a new one.
278    Configuration conf = new Configuration(baseConf);
279    CommonFSUtils.setRootDir(conf, rootDir);
280    CommonFSUtils.setWALRootDir(conf, walRootDir);
281    MasterRegionFlusherAndCompactor.setupConf(conf, params.flushSize(), params.flushPerChanges(),
282      params.flushIntervalMs());
283    conf.setInt(AbstractFSWAL.MAX_LOGS, params.maxWals());
284    if (params.useHsync() != null) {
285      conf.setBoolean(HRegion.WAL_HSYNC_CONF_KEY, params.useHsync());
286    }
287    if (params.useMetaCellComparator() != null) {
288      conf.setBoolean(HRegion.USE_META_CELL_COMPARATOR, params.useMetaCellComparator());
289    }
290    conf.setInt(AbstractFSWAL.RING_BUFFER_SLOT_COUNT,
291      IntMath.ceilingPowerOfTwo(params.ringBufferSlotCount()));
292
293    MasterRegionWALRoller walRoller = MasterRegionWALRoller.create(
294      td.getTableName() + "-WAL-Roller", conf, server, walFs, walRootDir, globalWALRootDir,
295      params.archivedWalSuffix(), params.rollPeriodMs(), params.flushSize());
296    walRoller.start();
297
298    WALFactory walFactory = new WALFactory(conf, server.getServerName().toString());
299    Path tableDir = CommonFSUtils.getTableDir(rootDir, td.getTableName());
300    HRegion region;
301    if (fs.exists(tableDir)) {
302      // load the existing region.
303      region = open(conf, td, fs, rootDir, walFs, walRootDir, walFactory, walRoller,
304        server.getServerName().toString());
305    } else {
306      // bootstrapping...
307      region = bootstrap(conf, td, fs, rootDir, walFs, walRootDir, walFactory, walRoller,
308        server.getServerName().toString());
309    }
310    Path globalArchiveDir = HFileArchiveUtil.getArchivePath(baseConf);
311    MasterRegionFlusherAndCompactor flusherAndCompactor = new MasterRegionFlusherAndCompactor(conf,
312      server, region, params.flushSize(), params.flushPerChanges(), params.flushIntervalMs(),
313      params.compactMin(), globalArchiveDir, params.archivedHFileSuffix());
314    walRoller.setFlusherAndCompactor(flusherAndCompactor);
315    Path archiveDir = HFileArchiveUtil.getArchivePath(conf);
316    if (!fs.mkdirs(archiveDir)) {
317      LOG.warn("Failed to create archive directory {}. Usually this should not happen but it will" +
318        " be created again when we actually archive the hfiles later, so continue", archiveDir);
319    }
320    return new MasterRegion(region, walFactory, flusherAndCompactor, walRoller);
321  }
322}