001/** 002 * 003 * Licensed to the Apache Software Foundation (ASF) under one 004 * or more contributor license agreements. See the NOTICE file 005 * distributed with this work for additional information 006 * regarding copyright ownership. The ASF licenses this file 007 * to you under the Apache License, Version 2.0 (the 008 * "License"); you may not use this file except in compliance 009 * with the License. You may obtain a copy of the License at 010 * 011 * http://www.apache.org/licenses/LICENSE-2.0 012 * 013 * Unless required by applicable law or agreed to in writing, software 014 * distributed under the License is distributed on an "AS IS" BASIS, 015 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 016 * See the License for the specific language governing permissions and 017 * limitations under the License. 018 */ 019package org.apache.hadoop.hbase.master; 020 021import java.io.IOException; 022 023import org.apache.hadoop.conf.Configuration; 024import org.apache.hadoop.fs.FileSystem; 025import org.apache.hadoop.fs.Path; 026import org.apache.hadoop.fs.permission.FsAction; 027import org.apache.hadoop.fs.permission.FsPermission; 028import org.apache.hadoop.hbase.ClusterId; 029import org.apache.hadoop.hbase.HConstants; 030import org.apache.hadoop.hbase.TableName; 031import org.apache.hadoop.hbase.backup.HFileArchiver; 032import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; 033import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; 034import org.apache.hadoop.hbase.client.RegionInfo; 035import org.apache.hadoop.hbase.client.RegionInfoBuilder; 036import org.apache.hadoop.hbase.client.TableDescriptor; 037import org.apache.hadoop.hbase.client.TableDescriptorBuilder; 038import org.apache.hadoop.hbase.exceptions.DeserializationException; 039import org.apache.hadoop.hbase.fs.HFileSystem; 040import org.apache.hadoop.hbase.log.HBaseMarkers; 041import org.apache.hadoop.hbase.mob.MobConstants; 042import org.apache.hadoop.hbase.procedure2.store.wal.WALProcedureStore; 043import org.apache.hadoop.hbase.regionserver.HRegion; 044import org.apache.hadoop.hbase.util.Bytes; 045import org.apache.hadoop.hbase.util.FSTableDescriptors; 046import org.apache.hadoop.hbase.util.FSUtils; 047import org.apache.hadoop.ipc.RemoteException; 048import org.apache.yetus.audience.InterfaceAudience; 049import org.slf4j.Logger; 050import org.slf4j.LoggerFactory; 051 052import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting; 053 054/** 055 * This class abstracts a bunch of operations the HMaster needs to interact with 056 * the underlying file system like creating the initial layout, checking file 057 * system status, etc. 058 */ 059@InterfaceAudience.Private 060public class MasterFileSystem { 061 private static final Logger LOG = LoggerFactory.getLogger(MasterFileSystem.class); 062 063 /** Parameter name for HBase instance root directory permission*/ 064 public static final String HBASE_DIR_PERMS = "hbase.rootdir.perms"; 065 066 /** Parameter name for HBase WAL directory permission*/ 067 public static final String HBASE_WAL_DIR_PERMS = "hbase.wal.dir.perms"; 068 069 // HBase configuration 070 private final Configuration conf; 071 // Persisted unique cluster ID 072 private ClusterId clusterId; 073 // Keep around for convenience. 074 private final FileSystem fs; 075 // Keep around for convenience. 076 private final FileSystem walFs; 077 // root log directory on the FS 078 private final Path rootdir; 079 // hbase temp directory used for table construction and deletion 080 private final Path tempdir; 081 // root hbase directory on the FS 082 private final Path walRootDir; 083 084 085 /* 086 * In a secure env, the protected sub-directories and files under the HBase rootDir 087 * would be restricted. The sub-directory will have '700' except the bulk load staging dir, 088 * which will have '711'. The default '700' can be overwritten by setting the property 089 * 'hbase.rootdir.perms'. The protected files (version file, clusterId file) will have '600'. 090 * The rootDir itself will be created with HDFS default permissions if it does not exist. 091 * We will check the rootDir permissions to make sure it has 'x' for all to ensure access 092 * to the staging dir. If it does not, we will add it. 093 */ 094 // Permissions for the directories under rootDir that need protection 095 private final FsPermission secureRootSubDirPerms; 096 // Permissions for the files under rootDir that need protection 097 private final FsPermission secureRootFilePerms = new FsPermission("600"); 098 // Permissions for bulk load staging directory under rootDir 099 private final FsPermission HiddenDirPerms = FsPermission.valueOf("-rwx--x--x"); 100 101 private boolean isSecurityEnabled; 102 103 public MasterFileSystem(Configuration conf) throws IOException { 104 this.conf = conf; 105 // Set filesystem to be that of this.rootdir else we get complaints about 106 // mismatched filesystems if hbase.rootdir is hdfs and fs.defaultFS is 107 // default localfs. Presumption is that rootdir is fully-qualified before 108 // we get to here with appropriate fs scheme. 109 this.rootdir = FSUtils.getRootDir(conf); 110 this.tempdir = new Path(this.rootdir, HConstants.HBASE_TEMP_DIRECTORY); 111 // Cover both bases, the old way of setting default fs and the new. 112 // We're supposed to run on 0.20 and 0.21 anyways. 113 this.fs = this.rootdir.getFileSystem(conf); 114 this.walRootDir = FSUtils.getWALRootDir(conf); 115 this.walFs = FSUtils.getWALFileSystem(conf); 116 FSUtils.setFsDefault(conf, new Path(this.walFs.getUri())); 117 walFs.setConf(conf); 118 FSUtils.setFsDefault(conf, new Path(this.fs.getUri())); 119 // make sure the fs has the same conf 120 fs.setConf(conf); 121 this.secureRootSubDirPerms = new FsPermission(conf.get("hbase.rootdir.perms", "700")); 122 this.isSecurityEnabled = "kerberos".equalsIgnoreCase(conf.get("hbase.security.authentication")); 123 // setup the filesystem variable 124 createInitialFileSystemLayout(); 125 HFileSystem.addLocationsOrderInterceptor(conf); 126 } 127 128 /** 129 * Create initial layout in filesystem. 130 * <ol> 131 * <li>Check if the meta region exists and is readable, if not create it. 132 * Create hbase.version and the hbase:meta directory if not one. 133 * </li> 134 * </ol> 135 * Idempotent. 136 */ 137 private void createInitialFileSystemLayout() throws IOException { 138 139 final String[] protectedSubDirs = new String[] { 140 HConstants.BASE_NAMESPACE_DIR, 141 HConstants.HFILE_ARCHIVE_DIRECTORY, 142 HConstants.HBCK_SIDELINEDIR_NAME, 143 MobConstants.MOB_DIR_NAME 144 }; 145 146 final String[] protectedSubLogDirs = new String[] { 147 HConstants.HREGION_LOGDIR_NAME, 148 HConstants.HREGION_OLDLOGDIR_NAME, 149 HConstants.CORRUPT_DIR_NAME, 150 WALProcedureStore.MASTER_PROCEDURE_LOGDIR 151 }; 152 // check if the root directory exists 153 checkRootDir(this.rootdir, conf, this.fs); 154 155 // Check the directories under rootdir. 156 checkTempDir(this.tempdir, conf, this.fs); 157 for (String subDir : protectedSubDirs) { 158 checkSubDir(new Path(this.rootdir, subDir), HBASE_DIR_PERMS); 159 } 160 161 final String perms; 162 if (!this.walRootDir.equals(this.rootdir)) { 163 perms = HBASE_WAL_DIR_PERMS; 164 } else { 165 perms = HBASE_DIR_PERMS; 166 } 167 for (String subDir : protectedSubLogDirs) { 168 checkSubDir(new Path(this.walRootDir, subDir), perms); 169 } 170 171 checkStagingDir(); 172 173 // Handle the last few special files and set the final rootDir permissions 174 // rootDir needs 'x' for all to support bulk load staging dir 175 if (isSecurityEnabled) { 176 fs.setPermission(new Path(rootdir, HConstants.VERSION_FILE_NAME), secureRootFilePerms); 177 fs.setPermission(new Path(rootdir, HConstants.CLUSTER_ID_FILE_NAME), secureRootFilePerms); 178 } 179 FsPermission currentRootPerms = fs.getFileStatus(this.rootdir).getPermission(); 180 if (!currentRootPerms.getUserAction().implies(FsAction.EXECUTE) 181 || !currentRootPerms.getGroupAction().implies(FsAction.EXECUTE) 182 || !currentRootPerms.getOtherAction().implies(FsAction.EXECUTE)) { 183 LOG.warn("rootdir permissions do not contain 'excute' for user, group or other. " 184 + "Automatically adding 'excute' permission for all"); 185 fs.setPermission( 186 this.rootdir, 187 new FsPermission(currentRootPerms.getUserAction().or(FsAction.EXECUTE), currentRootPerms 188 .getGroupAction().or(FsAction.EXECUTE), currentRootPerms.getOtherAction().or( 189 FsAction.EXECUTE))); 190 } 191 } 192 193 public FileSystem getFileSystem() { 194 return this.fs; 195 } 196 197 public FileSystem getWALFileSystem() { 198 return this.walFs; 199 } 200 201 public Configuration getConfiguration() { 202 return this.conf; 203 } 204 205 /** 206 * @return HBase root directory. 207 */ 208 public Path getRootDir() { 209 return this.rootdir; 210 } 211 212 /** 213 * @return HBase WAL root directory, usually the same as {@link #getRootDir()} but can be 214 * different if hfiles on one fs and WALs on another. The 'WALs' dir gets made underneath 215 * the root dir returned here; i.e. this is '/hbase', not '/hbase/WALs'. 216 */ 217 public Path getWALRootDir() { 218 return this.walRootDir; 219 } 220 221 /** 222 * @return the directory for a give {@code region}. 223 */ 224 public Path getRegionDir(RegionInfo region) { 225 return FSUtils.getRegionDirFromRootDir(getRootDir(), region); 226 } 227 228 /** 229 * @return HBase temp dir. 230 */ 231 public Path getTempDir() { 232 return this.tempdir; 233 } 234 235 /** 236 * @return The unique identifier generated for this cluster 237 */ 238 public ClusterId getClusterId() { 239 return clusterId; 240 } 241 242 /** 243 * Get the rootdir. Make sure its wholesome and exists before returning. 244 * @param rd 245 * @param c 246 * @param fs 247 * @return hbase.rootdir (after checks for existence and bootstrapping if 248 * needed populating the directory with necessary bootup files). 249 * @throws IOException 250 */ 251 private Path checkRootDir(final Path rd, final Configuration c, final FileSystem fs) 252 throws IOException { 253 // If FS is in safe mode wait till out of it. 254 FSUtils.waitOnSafeMode(c, c.getInt(HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000)); 255 256 // Filesystem is good. Go ahead and check for hbase.rootdir. 257 try { 258 if (!fs.exists(rd)) { 259 fs.mkdirs(rd); 260 // DFS leaves safe mode with 0 DNs when there are 0 blocks. 261 // We used to handle this by checking the current DN count and waiting until 262 // it is nonzero. With security, the check for datanode count doesn't work -- 263 // it is a privileged op. So instead we adopt the strategy of the jobtracker 264 // and simply retry file creation during bootstrap indefinitely. As soon as 265 // there is one datanode it will succeed. Permission problems should have 266 // already been caught by mkdirs above. 267 FSUtils.setVersion(fs, rd, c.getInt(HConstants.THREAD_WAKE_FREQUENCY, 268 10 * 1000), c.getInt(HConstants.VERSION_FILE_WRITE_ATTEMPTS, 269 HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS)); 270 } else { 271 if (!fs.isDirectory(rd)) { 272 throw new IllegalArgumentException(rd.toString() + " is not a directory"); 273 } 274 // as above 275 FSUtils.checkVersion(fs, rd, true, c.getInt(HConstants.THREAD_WAKE_FREQUENCY, 276 10 * 1000), c.getInt(HConstants.VERSION_FILE_WRITE_ATTEMPTS, 277 HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS)); 278 } 279 } catch (DeserializationException de) { 280 LOG.error(HBaseMarkers.FATAL, "Please fix invalid configuration for " 281 + HConstants.HBASE_DIR, de); 282 IOException ioe = new IOException(); 283 ioe.initCause(de); 284 throw ioe; 285 } catch (IllegalArgumentException iae) { 286 LOG.error(HBaseMarkers.FATAL, "Please fix invalid configuration for " 287 + HConstants.HBASE_DIR + " " + rd.toString(), iae); 288 throw iae; 289 } 290 // Make sure cluster ID exists 291 if (!FSUtils.checkClusterIdExists(fs, rd, c.getInt( 292 HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000))) { 293 FSUtils.setClusterId(fs, rd, new ClusterId(), c.getInt(HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000)); 294 } 295 clusterId = FSUtils.getClusterId(fs, rd); 296 297 // Make sure the meta region directory exists! 298 if (!FSUtils.metaRegionExists(fs, rd)) { 299 bootstrap(rd, c); 300 } 301 302 // Create tableinfo-s for hbase:meta if not already there. 303 // assume, created table descriptor is for enabling table 304 // meta table is a system table, so descriptors are predefined, 305 // we should get them from registry. 306 FSTableDescriptors fsd = new FSTableDescriptors(c, fs, rd); 307 fsd.createTableDescriptor(fsd.get(TableName.META_TABLE_NAME)); 308 309 return rd; 310 } 311 312 /** 313 * Make sure the hbase temp directory exists and is empty. 314 * NOTE that this method is only executed once just after the master becomes the active one. 315 */ 316 @VisibleForTesting 317 void checkTempDir(final Path tmpdir, final Configuration c, final FileSystem fs) 318 throws IOException { 319 // If the temp directory exists, clear the content (left over, from the previous run) 320 if (fs.exists(tmpdir)) { 321 // Archive table in temp, maybe left over from failed deletion, 322 // if not the cleaner will take care of them. 323 for (Path tableDir: FSUtils.getTableDirs(fs, tmpdir)) { 324 HFileArchiver.archiveRegions(c, fs, this.rootdir, tableDir, 325 FSUtils.getRegionDirs(fs, tableDir)); 326 } 327 if (!fs.delete(tmpdir, true)) { 328 throw new IOException("Unable to clean the temp directory: " + tmpdir); 329 } 330 } 331 332 // Create the temp directory 333 if (isSecurityEnabled) { 334 if (!fs.mkdirs(tmpdir, secureRootSubDirPerms)) { 335 throw new IOException("HBase temp directory '" + tmpdir + "' creation failure."); 336 } 337 } else { 338 if (!fs.mkdirs(tmpdir)) { 339 throw new IOException("HBase temp directory '" + tmpdir + "' creation failure."); 340 } 341 } 342 } 343 344 /** 345 * Make sure the directories under rootDir have good permissions. Create if necessary. 346 * @param p 347 * @throws IOException 348 */ 349 private void checkSubDir(final Path p, final String dirPermsConfName) throws IOException { 350 FileSystem fs = p.getFileSystem(conf); 351 FsPermission dirPerms = new FsPermission(conf.get(dirPermsConfName, "700")); 352 if (!fs.exists(p)) { 353 if (isSecurityEnabled) { 354 if (!fs.mkdirs(p, secureRootSubDirPerms)) { 355 throw new IOException("HBase directory '" + p + "' creation failure."); 356 } 357 } else { 358 if (!fs.mkdirs(p)) { 359 throw new IOException("HBase directory '" + p + "' creation failure."); 360 } 361 } 362 } 363 else { 364 if (isSecurityEnabled && !dirPerms.equals(fs.getFileStatus(p).getPermission())) { 365 // check whether the permission match 366 LOG.warn("Found HBase directory permissions NOT matching expected permissions for " 367 + p.toString() + " permissions=" + fs.getFileStatus(p).getPermission() 368 + ", expecting " + dirPerms + ". Automatically setting the permissions. " 369 + "You can change the permissions by setting \"" + dirPermsConfName + "\" in hbase-site.xml " 370 + "and restarting the master"); 371 fs.setPermission(p, dirPerms); 372 } 373 } 374 } 375 376 /** 377 * Check permissions for bulk load staging directory. This directory has special hidden 378 * permissions. Create it if necessary. 379 * @throws IOException 380 */ 381 private void checkStagingDir() throws IOException { 382 Path p = new Path(this.rootdir, HConstants.BULKLOAD_STAGING_DIR_NAME); 383 try { 384 if (!this.fs.exists(p)) { 385 if (!this.fs.mkdirs(p, HiddenDirPerms)) { 386 throw new IOException("Failed to create staging directory " + p.toString()); 387 } 388 } else { 389 this.fs.setPermission(p, HiddenDirPerms); 390 } 391 } catch (IOException e) { 392 LOG.error("Failed to create or set permission on staging directory " + p.toString()); 393 throw new IOException("Failed to create or set permission on staging directory " 394 + p.toString(), e); 395 } 396 } 397 398 private static void bootstrap(final Path rd, final Configuration c) 399 throws IOException { 400 LOG.info("BOOTSTRAP: creating hbase:meta region"); 401 try { 402 // Bootstrapping, make sure blockcache is off. Else, one will be 403 // created here in bootstrap and it'll need to be cleaned up. Better to 404 // not make it in first place. Turn off block caching for bootstrap. 405 // Enable after. 406 TableDescriptor metaDescriptor = new FSTableDescriptors(c).get(TableName.META_TABLE_NAME); 407 HRegion meta = HRegion.createHRegion(RegionInfoBuilder.FIRST_META_REGIONINFO, rd, 408 c, setInfoFamilyCachingForMeta(metaDescriptor, false), null); 409 meta.close(); 410 } catch (IOException e) { 411 e = e instanceof RemoteException ? 412 ((RemoteException)e).unwrapRemoteException() : e; 413 LOG.error("bootstrap", e); 414 throw e; 415 } 416 } 417 418 /** 419 * Enable in memory caching for hbase:meta 420 */ 421 public static TableDescriptor setInfoFamilyCachingForMeta(TableDescriptor metaDescriptor, final boolean b) { 422 TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(metaDescriptor); 423 for (ColumnFamilyDescriptor hcd: metaDescriptor.getColumnFamilies()) { 424 if (Bytes.equals(hcd.getName(), HConstants.CATALOG_FAMILY)) { 425 builder.modifyColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(hcd) 426 .setBlockCacheEnabled(b) 427 .setInMemory(b) 428 .build()); 429 } 430 } 431 return builder.build(); 432 } 433 434 public void deleteFamilyFromFS(RegionInfo region, byte[] familyName) 435 throws IOException { 436 deleteFamilyFromFS(rootdir, region, familyName); 437 } 438 439 public void deleteFamilyFromFS(Path rootDir, RegionInfo region, byte[] familyName) 440 throws IOException { 441 // archive family store files 442 Path tableDir = FSUtils.getTableDir(rootDir, region.getTable()); 443 HFileArchiver.archiveFamily(fs, conf, region, tableDir, familyName); 444 445 // delete the family folder 446 Path familyDir = new Path(tableDir, 447 new Path(region.getEncodedName(), Bytes.toString(familyName))); 448 if (fs.delete(familyDir, true) == false) { 449 if (fs.exists(familyDir)) { 450 throw new IOException("Could not delete family " 451 + Bytes.toString(familyName) + " from FileSystem for region " 452 + region.getRegionNameAsString() + "(" + region.getEncodedName() 453 + ")"); 454 } 455 } 456 } 457 458 public void stop() { 459 } 460 461 public void logFileSystemState(Logger log) throws IOException { 462 FSUtils.logFileSystemState(fs, rootdir, log); 463 } 464}