001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.master; 019 020import java.io.FileNotFoundException; 021import java.io.IOException; 022import org.apache.hadoop.conf.Configuration; 023import org.apache.hadoop.fs.FileStatus; 024import org.apache.hadoop.fs.FileSystem; 025import org.apache.hadoop.fs.Path; 026import org.apache.hadoop.fs.permission.FsAction; 027import org.apache.hadoop.fs.permission.FsPermission; 028import org.apache.hadoop.hbase.ClusterId; 029import org.apache.hadoop.hbase.HConstants; 030import org.apache.hadoop.hbase.backup.HFileArchiver; 031import org.apache.hadoop.hbase.client.RegionInfo; 032import org.apache.hadoop.hbase.exceptions.DeserializationException; 033import org.apache.hadoop.hbase.fs.HFileSystem; 034import org.apache.hadoop.hbase.log.HBaseMarkers; 035import org.apache.hadoop.hbase.mob.MobConstants; 036import org.apache.hadoop.hbase.security.access.SnapshotScannerHDFSAclHelper; 037import org.apache.hadoop.hbase.util.Bytes; 038import org.apache.hadoop.hbase.util.CommonFSUtils; 039import org.apache.hadoop.hbase.util.FSUtils; 040import org.apache.yetus.audience.InterfaceAudience; 041import org.slf4j.Logger; 042import org.slf4j.LoggerFactory; 043 044/** 045 * This class abstracts a bunch of operations the HMaster needs to interact with the underlying file 046 * system like creating the initial layout, checking file system status, etc. 047 */ 048@InterfaceAudience.Private 049public class MasterFileSystem { 050 private static final Logger LOG = LoggerFactory.getLogger(MasterFileSystem.class); 051 052 /** Parameter name for HBase instance root directory permission */ 053 public static final String HBASE_DIR_PERMS = "hbase.rootdir.perms"; 054 055 /** Parameter name for HBase WAL directory permission */ 056 public static final String HBASE_WAL_DIR_PERMS = "hbase.wal.dir.perms"; 057 058 // HBase configuration 059 private final Configuration conf; 060 // Persisted unique cluster ID 061 private ClusterId clusterId; 062 // Keep around for convenience. 063 private final FileSystem fs; 064 // Keep around for convenience. 065 private final FileSystem walFs; 066 // root log directory on the FS 067 private final Path rootdir; 068 // hbase temp directory used for table construction and deletion 069 private final Path tempdir; 070 // root hbase directory on the FS 071 private final Path walRootDir; 072 073 /* 074 * In a secure env, the protected sub-directories and files under the HBase rootDir would be 075 * restricted. The sub-directory will have '700' except the bulk load staging dir, which will have 076 * '711'. The default '700' can be overwritten by setting the property 'hbase.rootdir.perms'. The 077 * protected files (version file, clusterId file) will have '600'. The rootDir itself will be 078 * created with HDFS default permissions if it does not exist. We will check the rootDir 079 * permissions to make sure it has 'x' for all to ensure access to the staging dir. If it does 080 * not, we will add it. 081 */ 082 // Permissions for the directories under rootDir that need protection 083 private final FsPermission secureRootSubDirPerms; 084 // Permissions for the files under rootDir that need protection 085 private final FsPermission secureRootFilePerms = new FsPermission("600"); 086 // Permissions for bulk load staging directory under rootDir 087 private final FsPermission HiddenDirPerms = FsPermission.valueOf("-rwx--x--x"); 088 089 private boolean isSecurityEnabled; 090 091 public MasterFileSystem(Configuration conf) throws IOException { 092 this.conf = conf; 093 // Set filesystem to be that of this.rootdir else we get complaints about 094 // mismatched filesystems if hbase.rootdir is hdfs and fs.defaultFS is 095 // default localfs. Presumption is that rootdir is fully-qualified before 096 // we get to here with appropriate fs scheme. 097 this.rootdir = CommonFSUtils.getRootDir(conf); 098 this.tempdir = new Path(this.rootdir, HConstants.HBASE_TEMP_DIRECTORY); 099 // Cover both bases, the old way of setting default fs and the new. 100 // We're supposed to run on 0.20 and 0.21 anyways. 101 this.fs = this.rootdir.getFileSystem(conf); 102 this.walRootDir = CommonFSUtils.getWALRootDir(conf); 103 this.walFs = CommonFSUtils.getWALFileSystem(conf); 104 CommonFSUtils.setFsDefault(conf, new Path(this.walFs.getUri())); 105 walFs.setConf(conf); 106 CommonFSUtils.setFsDefault(conf, new Path(this.fs.getUri())); 107 // make sure the fs has the same conf 108 fs.setConf(conf); 109 this.secureRootSubDirPerms = new FsPermission(conf.get("hbase.rootdir.perms", "700")); 110 this.isSecurityEnabled = "kerberos".equalsIgnoreCase(conf.get("hbase.security.authentication")); 111 // setup the filesystem variable 112 createInitialFileSystemLayout(); 113 HFileSystem.addLocationsOrderInterceptor(conf); 114 } 115 116 /** 117 * Create initial layout in filesystem. 118 * <ol> 119 * <li>Check if the meta region exists and is readable, if not create it. Create hbase.version and 120 * the hbase:meta directory if not one.</li> 121 * </ol> 122 * Idempotent. 123 */ 124 private void createInitialFileSystemLayout() throws IOException { 125 126 final String[] protectedSubDirs = 127 new String[] { HConstants.BASE_NAMESPACE_DIR, HConstants.HFILE_ARCHIVE_DIRECTORY, 128 HConstants.HBCK_SIDELINEDIR_NAME, MobConstants.MOB_DIR_NAME }; 129 130 // With the introduction of RegionProcedureStore, 131 // there's no need to create MasterProcWAL dir here anymore. See HBASE-23715 132 final String[] protectedSubLogDirs = new String[] { HConstants.HREGION_LOGDIR_NAME, 133 HConstants.HREGION_OLDLOGDIR_NAME, HConstants.CORRUPT_DIR_NAME }; 134 // check if the root directory exists 135 checkRootDir(this.rootdir, conf, this.fs); 136 137 // Check the directories under rootdir. 138 checkTempDir(this.tempdir, conf, this.fs); 139 for (String subDir : protectedSubDirs) { 140 checkSubDir(new Path(this.rootdir, subDir), HBASE_DIR_PERMS); 141 } 142 143 final String perms; 144 if (!this.walRootDir.equals(this.rootdir)) { 145 perms = HBASE_WAL_DIR_PERMS; 146 } else { 147 perms = HBASE_DIR_PERMS; 148 } 149 for (String subDir : protectedSubLogDirs) { 150 checkSubDir(new Path(this.walRootDir, subDir), perms); 151 } 152 153 checkStagingDir(); 154 155 // Handle the last few special files and set the final rootDir permissions 156 // rootDir needs 'x' for all to support bulk load staging dir 157 if (isSecurityEnabled) { 158 fs.setPermission(new Path(rootdir, HConstants.VERSION_FILE_NAME), secureRootFilePerms); 159 fs.setPermission(new Path(rootdir, HConstants.CLUSTER_ID_FILE_NAME), secureRootFilePerms); 160 } 161 FsPermission currentRootPerms = fs.getFileStatus(this.rootdir).getPermission(); 162 if ( 163 !currentRootPerms.getUserAction().implies(FsAction.EXECUTE) 164 || !currentRootPerms.getGroupAction().implies(FsAction.EXECUTE) 165 || !currentRootPerms.getOtherAction().implies(FsAction.EXECUTE) 166 ) { 167 LOG.warn("rootdir permissions do not contain 'excute' for user, group or other. " 168 + "Automatically adding 'excute' permission for all"); 169 fs.setPermission(this.rootdir, 170 new FsPermission(currentRootPerms.getUserAction().or(FsAction.EXECUTE), 171 currentRootPerms.getGroupAction().or(FsAction.EXECUTE), 172 currentRootPerms.getOtherAction().or(FsAction.EXECUTE))); 173 } 174 } 175 176 public FileSystem getFileSystem() { 177 return this.fs; 178 } 179 180 public FileSystem getWALFileSystem() { 181 return this.walFs; 182 } 183 184 public Configuration getConfiguration() { 185 return this.conf; 186 } 187 188 /** Returns HBase root dir. */ 189 public Path getRootDir() { 190 return this.rootdir; 191 } 192 193 /** Returns HBase root log dir. */ 194 public Path getWALRootDir() { 195 return this.walRootDir; 196 } 197 198 /** Returns the directory for a give {@code region}. */ 199 public Path getRegionDir(RegionInfo region) { 200 return FSUtils.getRegionDirFromRootDir(getRootDir(), region); 201 } 202 203 /** Returns HBase temp dir. */ 204 public Path getTempDir() { 205 return this.tempdir; 206 } 207 208 /** Returns The unique identifier generated for this cluster */ 209 public ClusterId getClusterId() { 210 return clusterId; 211 } 212 213 /** 214 * Get the rootdir. Make sure its wholesome and exists before returning. nnn * @return 215 * hbase.rootdir (after checks for existence and bootstrapping if needed populating the directory 216 * with necessary bootup files). n 217 */ 218 private void checkRootDir(final Path rd, final Configuration c, final FileSystem fs) 219 throws IOException { 220 int threadWakeFrequency = c.getInt(HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000); 221 // If FS is in safe mode wait till out of it. 222 FSUtils.waitOnSafeMode(c, threadWakeFrequency); 223 224 // Filesystem is good. Go ahead and check for hbase.rootdir. 225 FileStatus status; 226 try { 227 status = fs.getFileStatus(rd); 228 } catch (FileNotFoundException e) { 229 status = null; 230 } 231 int versionFileWriteAttempts = c.getInt(HConstants.VERSION_FILE_WRITE_ATTEMPTS, 232 HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS); 233 try { 234 if (status == null) { 235 if (!fs.mkdirs(rd)) { 236 throw new IOException("Can not create configured '" + HConstants.HBASE_DIR + "' " + rd); 237 } 238 // DFS leaves safe mode with 0 DNs when there are 0 blocks. 239 // We used to handle this by checking the current DN count and waiting until 240 // it is nonzero. With security, the check for datanode count doesn't work -- 241 // it is a privileged op. So instead we adopt the strategy of the jobtracker 242 // and simply retry file creation during bootstrap indefinitely. As soon as 243 // there is one datanode it will succeed. Permission problems should have 244 // already been caught by mkdirs above. 245 FSUtils.setVersion(fs, rd, threadWakeFrequency, versionFileWriteAttempts); 246 } else { 247 if (!status.isDirectory()) { 248 throw new IllegalArgumentException( 249 "Configured '" + HConstants.HBASE_DIR + "' " + rd + " is not a directory."); 250 } 251 // as above 252 FSUtils.checkVersion(fs, rd, true, threadWakeFrequency, versionFileWriteAttempts); 253 } 254 } catch (DeserializationException de) { 255 LOG.error(HBaseMarkers.FATAL, "Please fix invalid configuration for '{}' {}", 256 HConstants.HBASE_DIR, rd, de); 257 throw new IOException(de); 258 } catch (IllegalArgumentException iae) { 259 LOG.error(HBaseMarkers.FATAL, "Please fix invalid configuration for '{}' {}", 260 HConstants.HBASE_DIR, rd, iae); 261 throw iae; 262 } 263 // Make sure cluster ID exists 264 if (!FSUtils.checkClusterIdExists(fs, rd, threadWakeFrequency)) { 265 FSUtils.setClusterId(fs, rd, new ClusterId(), threadWakeFrequency); 266 } 267 clusterId = FSUtils.getClusterId(fs, rd); 268 } 269 270 /** 271 * Make sure the hbase temp directory exists and is empty. NOTE that this method is only executed 272 * once just after the master becomes the active one. 273 */ 274 void checkTempDir(final Path tmpdir, final Configuration c, final FileSystem fs) 275 throws IOException { 276 // If the temp directory exists, clear the content (left over, from the previous run) 277 if (fs.exists(tmpdir)) { 278 // Archive table in temp, maybe left over from failed deletion, 279 // if not the cleaner will take care of them. 280 for (Path tableDir : FSUtils.getTableDirs(fs, tmpdir)) { 281 HFileArchiver.archiveRegions(c, fs, this.rootdir, tableDir, 282 FSUtils.getRegionDirs(fs, tableDir)); 283 if (!FSUtils.getRegionDirs(fs, tableDir).isEmpty()) { 284 LOG.warn("Found regions in tmp dir after archiving table regions, {}", tableDir); 285 } 286 } 287 // if acl sync to hdfs is enabled, then skip delete tmp dir because ACLs are set 288 if (!SnapshotScannerHDFSAclHelper.isAclSyncToHdfsEnabled(c) && !fs.delete(tmpdir, true)) { 289 throw new IOException("Unable to clean the temp directory: " + tmpdir); 290 } 291 } 292 293 // Create the temp directory 294 if (!fs.exists(tmpdir)) { 295 if (isSecurityEnabled) { 296 if (!fs.mkdirs(tmpdir, secureRootSubDirPerms)) { 297 throw new IOException("HBase temp directory '" + tmpdir + "' creation failure."); 298 } 299 } else { 300 if (!fs.mkdirs(tmpdir)) { 301 throw new IOException("HBase temp directory '" + tmpdir + "' creation failure."); 302 } 303 } 304 } 305 } 306 307 /** 308 * Make sure the directories under rootDir have good permissions. Create if necessary. nn 309 */ 310 private void checkSubDir(final Path p, final String dirPermsConfName) throws IOException { 311 FileSystem fs = p.getFileSystem(conf); 312 FsPermission dirPerms = new FsPermission(conf.get(dirPermsConfName, "700")); 313 if (!fs.exists(p)) { 314 if (isSecurityEnabled) { 315 if (!fs.mkdirs(p, secureRootSubDirPerms)) { 316 throw new IOException("HBase directory '" + p + "' creation failure."); 317 } 318 } else { 319 if (!fs.mkdirs(p)) { 320 throw new IOException("HBase directory '" + p + "' creation failure."); 321 } 322 } 323 } 324 if (isSecurityEnabled && !dirPerms.equals(fs.getFileStatus(p).getPermission())) { 325 // check whether the permission match 326 LOG.warn("Found HBase directory permissions NOT matching expected permissions for " 327 + p.toString() + " permissions=" + fs.getFileStatus(p).getPermission() + ", expecting " 328 + dirPerms + ". Automatically setting the permissions. " 329 + "You can change the permissions by setting \"" + dirPermsConfName 330 + "\" in hbase-site.xml " + "and restarting the master"); 331 fs.setPermission(p, dirPerms); 332 } 333 } 334 335 /** 336 * Check permissions for bulk load staging directory. This directory has special hidden 337 * permissions. Create it if necessary. n 338 */ 339 private void checkStagingDir() throws IOException { 340 Path p = new Path(this.rootdir, HConstants.BULKLOAD_STAGING_DIR_NAME); 341 try { 342 if (!this.fs.exists(p)) { 343 if (!this.fs.mkdirs(p, HiddenDirPerms)) { 344 throw new IOException("Failed to create staging directory " + p.toString()); 345 } 346 } 347 this.fs.setPermission(p, HiddenDirPerms); 348 349 } catch (IOException e) { 350 LOG.error("Failed to create or set permission on staging directory " + p.toString()); 351 throw new IOException( 352 "Failed to create or set permission on staging directory " + p.toString(), e); 353 } 354 } 355 356 public void deleteFamilyFromFS(RegionInfo region, byte[] familyName) throws IOException { 357 deleteFamilyFromFS(rootdir, region, familyName); 358 } 359 360 public void deleteFamilyFromFS(Path rootDir, RegionInfo region, byte[] familyName) 361 throws IOException { 362 // archive family store files 363 Path tableDir = CommonFSUtils.getTableDir(rootDir, region.getTable()); 364 HFileArchiver.archiveFamily(fs, conf, region, tableDir, familyName); 365 366 // delete the family folder 367 Path familyDir = 368 new Path(tableDir, new Path(region.getEncodedName(), Bytes.toString(familyName))); 369 if (fs.delete(familyDir, true) == false) { 370 if (fs.exists(familyDir)) { 371 throw new IOException( 372 "Could not delete family " + Bytes.toString(familyName) + " from FileSystem for region " 373 + region.getRegionNameAsString() + "(" + region.getEncodedName() + ")"); 374 } 375 } 376 } 377 378 public void stop() { 379 } 380 381 public void logFileSystemState(Logger log) throws IOException { 382 CommonFSUtils.logFileSystemState(fs, rootdir, log); 383 } 384}