001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master;
019
020import java.io.FileNotFoundException;
021import java.io.IOException;
022import org.apache.hadoop.conf.Configuration;
023import org.apache.hadoop.fs.FileStatus;
024import org.apache.hadoop.fs.FileSystem;
025import org.apache.hadoop.fs.Path;
026import org.apache.hadoop.fs.permission.FsAction;
027import org.apache.hadoop.fs.permission.FsPermission;
028import org.apache.hadoop.hbase.ClusterId;
029import org.apache.hadoop.hbase.HConstants;
030import org.apache.hadoop.hbase.backup.HFileArchiver;
031import org.apache.hadoop.hbase.client.RegionInfo;
032import org.apache.hadoop.hbase.exceptions.DeserializationException;
033import org.apache.hadoop.hbase.fs.HFileSystem;
034import org.apache.hadoop.hbase.log.HBaseMarkers;
035import org.apache.hadoop.hbase.mob.MobConstants;
036import org.apache.hadoop.hbase.replication.ReplicationUtils;
037import org.apache.hadoop.hbase.security.access.SnapshotScannerHDFSAclHelper;
038import org.apache.hadoop.hbase.util.Bytes;
039import org.apache.hadoop.hbase.util.CommonFSUtils;
040import org.apache.hadoop.hbase.util.FSUtils;
041import org.apache.yetus.audience.InterfaceAudience;
042import org.slf4j.Logger;
043import org.slf4j.LoggerFactory;
044
045/**
046 * This class abstracts a bunch of operations the HMaster needs to interact with the underlying file
047 * system like creating the initial layout, checking file system status, etc.
048 */
049@InterfaceAudience.Private
050public class MasterFileSystem {
051  private static final Logger LOG = LoggerFactory.getLogger(MasterFileSystem.class);
052
053  /** Parameter name for HBase instance root directory permission */
054  public static final String HBASE_DIR_PERMS = "hbase.rootdir.perms";
055
056  /** Parameter name for HBase WAL directory permission */
057  public static final String HBASE_WAL_DIR_PERMS = "hbase.wal.dir.perms";
058
059  // HBase configuration
060  private final Configuration conf;
061  // Persisted unique cluster ID
062  private ClusterId clusterId;
063  // Keep around for convenience.
064  private final FileSystem fs;
065  // Keep around for convenience.
066  private final FileSystem walFs;
067  // root log directory on the FS
068  private final Path rootdir;
069  // hbase temp directory used for table construction and deletion
070  private final Path tempdir;
071  // root hbase directory on the FS
072  private final Path walRootDir;
073
074  /*
075   * In a secure env, the protected sub-directories and files under the HBase rootDir would be
076   * restricted. The sub-directory will have '700' except the bulk load staging dir, which will have
077   * '711'. The default '700' can be overwritten by setting the property 'hbase.rootdir.perms'. The
078   * protected files (version file, clusterId file) will have '600'. The rootDir itself will be
079   * created with HDFS default permissions if it does not exist. We will check the rootDir
080   * permissions to make sure it has 'x' for all to ensure access to the staging dir. If it does
081   * not, we will add it.
082   */
083  // Permissions for the directories under rootDir that need protection
084  private final FsPermission secureRootSubDirPerms;
085  // Permissions for the files under rootDir that need protection
086  private final FsPermission secureRootFilePerms = new FsPermission("600");
087  // Permissions for bulk load staging directory under rootDir
088  private final FsPermission HiddenDirPerms = FsPermission.valueOf("-rwx--x--x");
089
090  private boolean isSecurityEnabled;
091
092  public MasterFileSystem(Configuration conf) throws IOException {
093    this.conf = conf;
094    // Set filesystem to be that of this.rootdir else we get complaints about
095    // mismatched filesystems if hbase.rootdir is hdfs and fs.defaultFS is
096    // default localfs. Presumption is that rootdir is fully-qualified before
097    // we get to here with appropriate fs scheme.
098    this.rootdir = CommonFSUtils.getRootDir(conf);
099    this.tempdir = new Path(this.rootdir, HConstants.HBASE_TEMP_DIRECTORY);
100    // Cover both bases, the old way of setting default fs and the new.
101    // We're supposed to run on 0.20 and 0.21 anyways.
102    this.fs = this.rootdir.getFileSystem(conf);
103    this.walRootDir = CommonFSUtils.getWALRootDir(conf);
104    this.walFs = CommonFSUtils.getWALFileSystem(conf);
105    CommonFSUtils.setFsDefault(conf, new Path(this.walFs.getUri()));
106    walFs.setConf(conf);
107    CommonFSUtils.setFsDefault(conf, new Path(this.fs.getUri()));
108    // make sure the fs has the same conf
109    fs.setConf(conf);
110    this.secureRootSubDirPerms = new FsPermission(conf.get("hbase.rootdir.perms", "700"));
111    this.isSecurityEnabled = "kerberos".equalsIgnoreCase(conf.get("hbase.security.authentication"));
112    // setup the filesystem variable
113    createInitialFileSystemLayout();
114    HFileSystem.addLocationsOrderInterceptor(conf);
115  }
116
117  /**
118   * Create initial layout in filesystem.
119   * <ol>
120   * <li>Check if the meta region exists and is readable, if not create it. Create hbase.version and
121   * the hbase:meta directory if not one.</li>
122   * </ol>
123   * Idempotent.
124   */
125  private void createInitialFileSystemLayout() throws IOException {
126    final String[] protectedSubDirs =
127      new String[] { HConstants.BASE_NAMESPACE_DIR, HConstants.HFILE_ARCHIVE_DIRECTORY,
128        HConstants.HBCK_SIDELINEDIR_NAME, MobConstants.MOB_DIR_NAME };
129
130    // With the introduction of RegionProcedureStore,
131    // there's no need to create MasterProcWAL dir here anymore. See HBASE-23715
132    final String[] protectedSubLogDirs =
133      new String[] { HConstants.HREGION_LOGDIR_NAME, HConstants.HREGION_OLDLOGDIR_NAME,
134        HConstants.CORRUPT_DIR_NAME, ReplicationUtils.REMOTE_WAL_DIR_NAME };
135    // check if the root directory exists
136    checkRootDir(this.rootdir, conf, this.fs);
137
138    // Check the directories under rootdir.
139    checkTempDir(this.tempdir, conf, this.fs);
140    for (String subDir : protectedSubDirs) {
141      checkSubDir(new Path(this.rootdir, subDir), HBASE_DIR_PERMS);
142    }
143
144    final String perms;
145    if (!this.walRootDir.equals(this.rootdir)) {
146      perms = HBASE_WAL_DIR_PERMS;
147    } else {
148      perms = HBASE_DIR_PERMS;
149    }
150    for (String subDir : protectedSubLogDirs) {
151      checkSubDir(new Path(this.walRootDir, subDir), perms);
152    }
153
154    checkStagingDir();
155
156    // Handle the last few special files and set the final rootDir permissions
157    // rootDir needs 'x' for all to support bulk load staging dir
158    if (isSecurityEnabled) {
159      fs.setPermission(new Path(rootdir, HConstants.VERSION_FILE_NAME), secureRootFilePerms);
160      fs.setPermission(new Path(rootdir, HConstants.CLUSTER_ID_FILE_NAME), secureRootFilePerms);
161    }
162    FsPermission currentRootPerms = fs.getFileStatus(this.rootdir).getPermission();
163    if (
164      !currentRootPerms.getUserAction().implies(FsAction.EXECUTE)
165        || !currentRootPerms.getGroupAction().implies(FsAction.EXECUTE)
166        || !currentRootPerms.getOtherAction().implies(FsAction.EXECUTE)
167    ) {
168      LOG.warn("rootdir permissions do not contain 'excute' for user, group or other. "
169        + "Automatically adding 'excute' permission for all");
170      fs.setPermission(this.rootdir,
171        new FsPermission(currentRootPerms.getUserAction().or(FsAction.EXECUTE),
172          currentRootPerms.getGroupAction().or(FsAction.EXECUTE),
173          currentRootPerms.getOtherAction().or(FsAction.EXECUTE)));
174    }
175  }
176
177  public FileSystem getFileSystem() {
178    return this.fs;
179  }
180
181  public FileSystem getWALFileSystem() {
182    return this.walFs;
183  }
184
185  public Configuration getConfiguration() {
186    return this.conf;
187  }
188
189  /** Returns HBase root dir. */
190  public Path getRootDir() {
191    return this.rootdir;
192  }
193
194  /** Returns HBase root log dir. */
195  public Path getWALRootDir() {
196    return this.walRootDir;
197  }
198
199  /** Returns the directory for a give {@code region}. */
200  public Path getRegionDir(RegionInfo region) {
201    return FSUtils.getRegionDirFromRootDir(getRootDir(), region);
202  }
203
204  /** Returns HBase temp dir. */
205  public Path getTempDir() {
206    return this.tempdir;
207  }
208
209  /** Returns The unique identifier generated for this cluster */
210  public ClusterId getClusterId() {
211    return clusterId;
212  }
213
214  /**
215   * Get the rootdir. Make sure its wholesome and exists before returning.
216   * @return hbase.rootdir (after checks for existence and bootstrapping if needed populating the
217   *         directory with necessary bootup files).
218   */
219  private void checkRootDir(final Path rd, final Configuration c, final FileSystem fs)
220    throws IOException {
221    int threadWakeFrequency = c.getInt(HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000);
222    // If FS is in safe mode wait till out of it.
223    FSUtils.waitOnSafeMode(c, threadWakeFrequency);
224
225    // Filesystem is good. Go ahead and check for hbase.rootdir.
226    FileStatus status;
227    try {
228      status = fs.getFileStatus(rd);
229    } catch (FileNotFoundException e) {
230      status = null;
231    }
232    int versionFileWriteAttempts = c.getInt(HConstants.VERSION_FILE_WRITE_ATTEMPTS,
233      HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS);
234    try {
235      if (status == null) {
236        if (!fs.mkdirs(rd)) {
237          throw new IOException("Can not create configured '" + HConstants.HBASE_DIR + "' " + rd);
238        }
239        // DFS leaves safe mode with 0 DNs when there are 0 blocks.
240        // We used to handle this by checking the current DN count and waiting until
241        // it is nonzero. With security, the check for datanode count doesn't work --
242        // it is a privileged op. So instead we adopt the strategy of the jobtracker
243        // and simply retry file creation during bootstrap indefinitely. As soon as
244        // there is one datanode it will succeed. Permission problems should have
245        // already been caught by mkdirs above.
246        FSUtils.setVersion(fs, rd, threadWakeFrequency, versionFileWriteAttempts);
247      } else {
248        if (!status.isDirectory()) {
249          throw new IllegalArgumentException(
250            "Configured '" + HConstants.HBASE_DIR + "' " + rd + " is not a directory.");
251        }
252        // as above
253        FSUtils.checkVersion(fs, rd, true, threadWakeFrequency, versionFileWriteAttempts);
254      }
255    } catch (DeserializationException de) {
256      LOG.error(HBaseMarkers.FATAL, "Please fix invalid configuration for '{}' {}",
257        HConstants.HBASE_DIR, rd, de);
258      throw new IOException(de);
259    } catch (IllegalArgumentException iae) {
260      LOG.error(HBaseMarkers.FATAL, "Please fix invalid configuration for '{}' {}",
261        HConstants.HBASE_DIR, rd, iae);
262      throw iae;
263    }
264    // Make sure cluster ID exists
265    if (!FSUtils.checkClusterIdExists(fs, rd, threadWakeFrequency)) {
266      FSUtils.setClusterId(fs, rd, new ClusterId(), threadWakeFrequency);
267    }
268    clusterId = FSUtils.getClusterId(fs, rd);
269  }
270
271  /**
272   * Make sure the hbase temp directory exists and is empty. NOTE that this method is only executed
273   * once just after the master becomes the active one.
274   */
275  void checkTempDir(final Path tmpdir, final Configuration c, final FileSystem fs)
276    throws IOException {
277    // If the temp directory exists, clear the content (left over, from the previous run)
278    if (fs.exists(tmpdir)) {
279      // Archive table in temp, maybe left over from failed deletion,
280      // if not the cleaner will take care of them.
281      for (Path tableDir : FSUtils.getTableDirs(fs, tmpdir)) {
282        HFileArchiver.archiveRegions(c, fs, this.rootdir, tableDir,
283          FSUtils.getRegionDirs(fs, tableDir));
284        if (!FSUtils.getRegionDirs(fs, tableDir).isEmpty()) {
285          LOG.warn("Found regions in tmp dir after archiving table regions, {}", tableDir);
286        }
287      }
288      // if acl sync to hdfs is enabled, then skip delete tmp dir because ACLs are set
289      if (!SnapshotScannerHDFSAclHelper.isAclSyncToHdfsEnabled(c) && !fs.delete(tmpdir, true)) {
290        throw new IOException("Unable to clean the temp directory: " + tmpdir);
291      }
292    }
293
294    // Create the temp directory
295    if (!fs.exists(tmpdir)) {
296      if (isSecurityEnabled) {
297        if (!fs.mkdirs(tmpdir, secureRootSubDirPerms)) {
298          throw new IOException("HBase temp directory '" + tmpdir + "' creation failure.");
299        }
300      } else {
301        if (!fs.mkdirs(tmpdir)) {
302          throw new IOException("HBase temp directory '" + tmpdir + "' creation failure.");
303        }
304      }
305    }
306  }
307
308  /**
309   * Make sure the directories under rootDir have good permissions. Create if necessary.
310   */
311  private void checkSubDir(final Path p, final String dirPermsConfName) throws IOException {
312    FileSystem fs = p.getFileSystem(conf);
313    FsPermission dirPerms = new FsPermission(conf.get(dirPermsConfName, "700"));
314    if (!fs.exists(p)) {
315      if (isSecurityEnabled) {
316        if (!fs.mkdirs(p, secureRootSubDirPerms)) {
317          throw new IOException("HBase directory '" + p + "' creation failure.");
318        }
319      } else {
320        if (!fs.mkdirs(p)) {
321          throw new IOException("HBase directory '" + p + "' creation failure.");
322        }
323      }
324    }
325    if (isSecurityEnabled && !dirPerms.equals(fs.getFileStatus(p).getPermission())) {
326      // check whether the permission match
327      LOG.warn("Found HBase directory permissions NOT matching expected permissions for "
328        + p.toString() + " permissions=" + fs.getFileStatus(p).getPermission() + ", expecting "
329        + dirPerms + ". Automatically setting the permissions. "
330        + "You can change the permissions by setting \"" + dirPermsConfName
331        + "\" in hbase-site.xml " + "and restarting the master");
332      fs.setPermission(p, dirPerms);
333    }
334  }
335
336  /**
337   * Check permissions for bulk load staging directory. This directory has special hidden
338   * permissions. Create it if necessary.
339   */
340  private void checkStagingDir() throws IOException {
341    Path p = new Path(this.rootdir, HConstants.BULKLOAD_STAGING_DIR_NAME);
342    try {
343      if (!this.fs.exists(p)) {
344        if (!this.fs.mkdirs(p, HiddenDirPerms)) {
345          throw new IOException("Failed to create staging directory " + p.toString());
346        }
347      }
348      this.fs.setPermission(p, HiddenDirPerms);
349
350    } catch (IOException e) {
351      LOG.error("Failed to create or set permission on staging directory " + p.toString());
352      throw new IOException(
353        "Failed to create or set permission on staging directory " + p.toString(), e);
354    }
355  }
356
357  public void deleteFamilyFromFS(RegionInfo region, byte[] familyName) throws IOException {
358    deleteFamilyFromFS(rootdir, region, familyName);
359  }
360
361  public void deleteFamilyFromFS(Path rootDir, RegionInfo region, byte[] familyName)
362    throws IOException {
363    // archive family store files
364    Path tableDir = CommonFSUtils.getTableDir(rootDir, region.getTable());
365    HFileArchiver.archiveFamily(fs, conf, region, tableDir, familyName);
366
367    // delete the family folder
368    Path familyDir =
369      new Path(tableDir, new Path(region.getEncodedName(), Bytes.toString(familyName)));
370    if (fs.delete(familyDir, true) == false) {
371      if (fs.exists(familyDir)) {
372        throw new IOException(
373          "Could not delete family " + Bytes.toString(familyName) + " from FileSystem for region "
374            + region.getRegionNameAsString() + "(" + region.getEncodedName() + ")");
375      }
376    }
377  }
378
379  public void stop() {
380  }
381
382  public void logFileSystemState(Logger log) throws IOException {
383    CommonFSUtils.logFileSystemState(fs, rootdir, log);
384  }
385}