View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.master;
20  
21  import java.io.IOException;
22
23  import org.apache.commons.logging.Log;
24  import org.apache.commons.logging.LogFactory;
25  import org.apache.hadoop.conf.Configuration;
26  import org.apache.hadoop.fs.FileSystem;
27  import org.apache.hadoop.fs.Path;
28  import org.apache.hadoop.fs.permission.FsAction;
29  import org.apache.hadoop.fs.permission.FsPermission;
30  import org.apache.hadoop.hbase.ClusterId;
31  import org.apache.hadoop.hbase.HColumnDescriptor;
32  import org.apache.hadoop.hbase.HConstants;
33  import org.apache.hadoop.hbase.HRegionInfo;
34  import org.apache.hadoop.hbase.HTableDescriptor;
35  import org.apache.hadoop.hbase.TableName;
36  import org.apache.hadoop.hbase.backup.HFileArchiver;
37  import org.apache.hadoop.hbase.classification.InterfaceAudience;
38  import org.apache.hadoop.hbase.exceptions.DeserializationException;
39  import org.apache.hadoop.hbase.fs.HFileSystem;
40  import org.apache.hadoop.hbase.master.procedure.MasterProcedureConstants;
41  import org.apache.hadoop.hbase.mob.MobConstants;
42  import org.apache.hadoop.hbase.mob.MobUtils;
43  import org.apache.hadoop.hbase.regionserver.HRegion;
44  import org.apache.hadoop.hbase.util.Bytes;
45  import org.apache.hadoop.hbase.util.FSTableDescriptors;
46  import org.apache.hadoop.hbase.util.FSUtils;
47  import org.apache.hadoop.ipc.RemoteException;
48
49  /**
50   * This class abstracts a bunch of operations the HMaster needs to interact with
51   * the underlying file system like creating the initial layout, checking file
52   * system status, etc.
53   */
54  @InterfaceAudience.Private
55  public class MasterFileSystem {
56    private static final Log LOG = LogFactory.getLog(MasterFileSystem.class);
57
58    // HBase configuration
59    private final Configuration conf;
60    // Persisted unique cluster ID
61    private ClusterId clusterId;
62    // Keep around for convenience.
63    private final FileSystem fs;
64    // root hbase directory on the FS
65    private final Path rootdir;
66    // hbase temp directory used for table construction and deletion
67    private final Path tempdir;
68
69
70    /*
71     * In a secure env, the protected sub-directories and files under the HBase rootDir
72     * would be restricted. The sub-directory will have '700' except the bulk load staging dir,
73     * which will have '711'.  The default '700' can be overwritten by setting the property
74     * 'hbase.rootdir.perms'. The protected files (version file, clusterId file) will have '600'.
75     * The rootDir itself will be created with HDFS default permissions if it does not exist.
76     * We will check the rootDir permissions to make sure it has 'x' for all to ensure access
77     * to the staging dir. If it does not, we will add it.
78     */
79    // Permissions for the directories under rootDir that need protection
80    private final FsPermission secureRootSubDirPerms;
81    // Permissions for the files under rootDir that need protection
82    private final FsPermission secureRootFilePerms = new FsPermission("600");
83    // Permissions for bulk load staging directory under rootDir
84    private final FsPermission HiddenDirPerms = FsPermission.valueOf("-rwx--x--x");
85
86    private boolean isSecurityEnabled;
87
88    private final MasterServices services;
89
90    public MasterFileSystem(MasterServices services) throws IOException {
91      this.conf = services.getConfiguration();
92      this.services = services;
93      // Set filesystem to be that of this.rootdir else we get complaints about
94      // mismatched filesystems if hbase.rootdir is hdfs and fs.defaultFS is
95      // default localfs.  Presumption is that rootdir is fully-qualified before
96      // we get to here with appropriate fs scheme.
97      this.rootdir = FSUtils.getRootDir(conf);
98      this.tempdir = new Path(this.rootdir, HConstants.HBASE_TEMP_DIRECTORY);
99      // Cover both bases, the old way of setting default fs and the new.
100     // We're supposed to run on 0.20 and 0.21 anyways.
101     this.fs = this.rootdir.getFileSystem(conf);
102     FSUtils.setFsDefault(conf, new Path(this.fs.getUri()));
103     // make sure the fs has the same conf
104     fs.setConf(conf);
105     this.secureRootSubDirPerms = new FsPermission(conf.get("hbase.rootdir.perms", "700"));
106     this.isSecurityEnabled = "kerberos".equalsIgnoreCase(conf.get("hbase.security.authentication"));
107     // setup the filesystem variable
108     createInitialFileSystemLayout();
109     HFileSystem.addLocationsOrderInterceptor(conf);
110   }
111
112   /**
113    * Create initial layout in filesystem.
114    * <ol>
115    * <li>Check if the meta region exists and is readable, if not create it.
116    * Create hbase.version and the hbase:meta directory if not one.
117    * </li>
118    * </ol>
119    * Idempotent.
120    */
121   private void createInitialFileSystemLayout() throws IOException {
122
123     final String[] protectedSubDirs = new String[] {
124         HConstants.BASE_NAMESPACE_DIR,
125         HConstants.HFILE_ARCHIVE_DIRECTORY,
126         HConstants.HREGION_LOGDIR_NAME,
127         HConstants.HREGION_OLDLOGDIR_NAME,
128         MasterProcedureConstants.MASTER_PROCEDURE_LOGDIR,
129         HConstants.CORRUPT_DIR_NAME,
130         HConstants.HBCK_SIDELINEDIR_NAME,
131         MobConstants.MOB_DIR_NAME
132     };
133     // check if the root directory exists
134     checkRootDir(this.rootdir, conf, this.fs);
135
136     // Check the directories under rootdir.
137     checkTempDir(this.tempdir, conf, this.fs);
138     for (String subDir : protectedSubDirs) {
139       checkSubDir(new Path(this.rootdir, subDir));
140     }
141
142     checkStagingDir();
143
144     // Handle the last few special files and set the final rootDir permissions
145     // rootDir needs 'x' for all to support bulk load staging dir
146     if (isSecurityEnabled) {
147       fs.setPermission(new Path(rootdir, HConstants.VERSION_FILE_NAME), secureRootFilePerms);
148       fs.setPermission(new Path(rootdir, HConstants.CLUSTER_ID_FILE_NAME), secureRootFilePerms);
149     }
150     FsPermission currentRootPerms = fs.getFileStatus(this.rootdir).getPermission();
151     if (!currentRootPerms.getUserAction().implies(FsAction.EXECUTE)
152         || !currentRootPerms.getGroupAction().implies(FsAction.EXECUTE)
153         || !currentRootPerms.getOtherAction().implies(FsAction.EXECUTE)) {
154       LOG.warn("rootdir permissions do not contain 'excute' for user, group or other. "
155         + "Automatically adding 'excute' permission for all");
156       fs.setPermission(
157         this.rootdir,
158         new FsPermission(currentRootPerms.getUserAction().or(FsAction.EXECUTE), currentRootPerms
159             .getGroupAction().or(FsAction.EXECUTE), currentRootPerms.getOtherAction().or(
160           FsAction.EXECUTE)));
161     }
162   }
163
164   public FileSystem getFileSystem() {
165     return this.fs;
166   }
167
168   public Configuration getConfiguration() {
169     return this.conf;
170   }
171
172   /**
173    * @return HBase root dir.
174    */
175   public Path getRootDir() {
176     return this.rootdir;
177   }
178
179   /**
180    * @return HBase temp dir.
181    */
182   public Path getTempDir() {
183     return this.tempdir;
184   }
185
186   /**
187    * @return The unique identifier generated for this cluster
188    */
189   public ClusterId getClusterId() {
190     return clusterId;
191   }
192
193   /**
194    * Get the rootdir.  Make sure its wholesome and exists before returning.
195    * @param rd
196    * @param c
197    * @param fs
198    * @return hbase.rootdir (after checks for existence and bootstrapping if
199    * needed populating the directory with necessary bootup files).
200    * @throws IOException
201    */
202   private Path checkRootDir(final Path rd, final Configuration c, final FileSystem fs)
203       throws IOException {
204     // If FS is in safe mode wait till out of it.
205     FSUtils.waitOnSafeMode(c, c.getInt(HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000));
206
207     // Filesystem is good. Go ahead and check for hbase.rootdir.
208     try {
209       if (!fs.exists(rd)) {
210         fs.mkdirs(rd);
211         // DFS leaves safe mode with 0 DNs when there are 0 blocks.
212         // We used to handle this by checking the current DN count and waiting until
213         // it is nonzero. With security, the check for datanode count doesn't work --
214         // it is a privileged op. So instead we adopt the strategy of the jobtracker
215         // and simply retry file creation during bootstrap indefinitely. As soon as
216         // there is one datanode it will succeed. Permission problems should have
217         // already been caught by mkdirs above.
218         FSUtils.setVersion(fs, rd, c.getInt(HConstants.THREAD_WAKE_FREQUENCY,
219           10 * 1000), c.getInt(HConstants.VERSION_FILE_WRITE_ATTEMPTS,
220             HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS));
221       } else {
222         if (!fs.isDirectory(rd)) {
223           throw new IllegalArgumentException(rd.toString() + " is not a directory");
224         }
225         // as above
226         FSUtils.checkVersion(fs, rd, true, c.getInt(HConstants.THREAD_WAKE_FREQUENCY,
227           10 * 1000), c.getInt(HConstants.VERSION_FILE_WRITE_ATTEMPTS,
228             HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS));
229       }
230     } catch (DeserializationException de) {
231       LOG.fatal("Please fix invalid configuration for " + HConstants.HBASE_DIR, de);
232       IOException ioe = new IOException();
233       ioe.initCause(de);
234       throw ioe;
235     } catch (IllegalArgumentException iae) {
236       LOG.fatal("Please fix invalid configuration for "
237         + HConstants.HBASE_DIR + " " + rd.toString(), iae);
238       throw iae;
239     }
240     // Make sure cluster ID exists
241     if (!FSUtils.checkClusterIdExists(fs, rd, c.getInt(
242         HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000))) {
243       FSUtils.setClusterId(fs, rd, new ClusterId(), c.getInt(HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000));
244     }
245     clusterId = FSUtils.getClusterId(fs, rd);
246
247     // Make sure the meta region directory exists!
248     if (!FSUtils.metaRegionExists(fs, rd)) {
249       bootstrap(rd, c);
250     }
251
252     // Create tableinfo-s for hbase:meta if not already there.
253     // assume, created table descriptor is for enabling table
254     // meta table is a system table, so descriptors are predefined,
255     // we should get them from registry.
256     FSTableDescriptors fsd = new FSTableDescriptors(c, fs, rd);
257     fsd.createTableDescriptor(fsd.get(TableName.META_TABLE_NAME));
258
259     return rd;
260   }
261
262   /**
263    * Make sure the hbase temp directory exists and is empty.
264    * NOTE that this method is only executed once just after the master becomes the active one.
265    */
266   private void checkTempDir(final Path tmpdir, final Configuration c, final FileSystem fs)
267       throws IOException {
268     // If the temp directory exists, clear the content (left over, from the previous run)
269     if (fs.exists(tmpdir)) {
270       // Archive table in temp, maybe left over from failed deletion,
271       // if not the cleaner will take care of them.
272       for (Path tabledir: FSUtils.getTableDirs(fs, tmpdir)) {
273         for (Path regiondir: FSUtils.getRegionDirs(fs, tabledir)) {
274           HFileArchiver.archiveRegion(fs, this.rootdir, tabledir, regiondir);
275         }
276       }
277       if (!fs.delete(tmpdir, true)) {
278         throw new IOException("Unable to clean the temp directory: " + tmpdir);
279       }
280     }
281
282     // Create the temp directory
283     if (isSecurityEnabled) {
284       if (!fs.mkdirs(tmpdir, secureRootSubDirPerms)) {
285         throw new IOException("HBase temp directory '" + tmpdir + "' creation failure.");
286       }
287     } else {
288       if (!fs.mkdirs(tmpdir)) {
289         throw new IOException("HBase temp directory '" + tmpdir + "' creation failure.");
290       }
291     }
292   }
293
294   /**
295    * Make sure the directories under rootDir have good permissions. Create if necessary.
296    * @param p
297    * @throws IOException
298    */
299   private void checkSubDir(final Path p) throws IOException {
300     if (!fs.exists(p)) {
301       if (isSecurityEnabled) {
302         if (!fs.mkdirs(p, secureRootSubDirPerms)) {
303           throw new IOException("HBase directory '" + p + "' creation failure.");
304         }
305       } else {
306         if (!fs.mkdirs(p)) {
307           throw new IOException("HBase directory '" + p + "' creation failure.");
308         }
309       }
310     }
311     else {
312       if (isSecurityEnabled && !secureRootSubDirPerms.equals(fs.getFileStatus(p).getPermission())) {
313         // check whether the permission match
314         LOG.warn("Found HBase directory permissions NOT matching expected permissions for "
315             + p.toString() + " permissions=" + fs.getFileStatus(p).getPermission()
316             + ", expecting " + secureRootSubDirPerms + ". Automatically setting the permissions. "
317             + "You can change the permissions by setting \"hbase.rootdir.perms\" in hbase-site.xml "
318             + "and restarting the master");
319         fs.setPermission(p, secureRootSubDirPerms);
320       }
321     }
322   }
323
324   /**
325    * Check permissions for bulk load staging directory. This directory has special hidden
326    * permissions. Create it if necessary.
327    * @throws IOException
328    */
329   private void checkStagingDir() throws IOException {
330     Path p = new Path(this.rootdir, HConstants.BULKLOAD_STAGING_DIR_NAME);
331     try {
332       if (!this.fs.exists(p)) {
333         if (!this.fs.mkdirs(p, HiddenDirPerms)) {
334           throw new IOException("Failed to create staging directory " + p.toString());
335         }
336       } else {
337         this.fs.setPermission(p, HiddenDirPerms);
338       }
339     } catch (IOException e) {
340       LOG.error("Failed to create or set permission on staging directory " + p.toString());
341       throw new IOException("Failed to create or set permission on staging directory "
342           + p.toString(), e);
343     }
344   }
345
346   private static void bootstrap(final Path rd, final Configuration c)
347   throws IOException {
348     LOG.info("BOOTSTRAP: creating hbase:meta region");
349     try {
350       // Bootstrapping, make sure blockcache is off.  Else, one will be
351       // created here in bootstrap and it'll need to be cleaned up.  Better to
352       // not make it in first place.  Turn off block caching for bootstrap.
353       // Enable after.
354       HRegionInfo metaHRI = new HRegionInfo(HRegionInfo.FIRST_META_REGIONINFO);
355       HTableDescriptor metaDescriptor = new FSTableDescriptors(c).get(TableName.META_TABLE_NAME);
356       setInfoFamilyCachingForMeta(metaDescriptor, false);
357       HRegion meta = HRegion.createHRegion(metaHRI, rd, c, metaDescriptor, null);
358       setInfoFamilyCachingForMeta(metaDescriptor, true);
359       meta.close();
360     } catch (IOException e) {
361         e = e instanceof RemoteException ?
362                 ((RemoteException)e).unwrapRemoteException() : e;
363       LOG.error("bootstrap", e);
364       throw e;
365     }
366   }
367
368   /**
369    * Enable in memory caching for hbase:meta
370    */
371   public static void setInfoFamilyCachingForMeta(HTableDescriptor metaDescriptor, final boolean b) {
372     for (HColumnDescriptor hcd: metaDescriptor.getColumnFamilies()) {
373       if (Bytes.equals(hcd.getName(), HConstants.CATALOG_FAMILY)) {
374         hcd.setBlockCacheEnabled(b);
375         hcd.setInMemory(b);
376       }
377     }
378   }
379
380   public void deleteFamilyFromFS(HRegionInfo region, byte[] familyName, boolean hasMob)
381       throws IOException {
382     // archive family store files
383     Path tableDir = FSUtils.getTableDir(rootdir, region.getTable());
384     HFileArchiver.archiveFamily(fs, conf, region, tableDir, familyName);
385
386     // delete the family folder
387     Path familyDir = new Path(tableDir,
388       new Path(region.getEncodedName(), Bytes.toString(familyName)));
389     if (fs.delete(familyDir, true) == false) {
390       if (fs.exists(familyDir)) {
391         throw new IOException("Could not delete family "
392             + Bytes.toString(familyName) + " from FileSystem for region "
393             + region.getRegionNameAsString() + "(" + region.getEncodedName()
394             + ")");
395       }
396     }
397
398     // archive and delete mob files
399     if (hasMob) {
400       Path mobTableDir =
401           FSUtils.getTableDir(new Path(getRootDir(), MobConstants.MOB_DIR_NAME), region.getTable());
402       HRegionInfo mobRegionInfo = MobUtils.getMobRegionInfo(region.getTable());
403       Path mobFamilyDir =
404           new Path(mobTableDir,
405               new Path(mobRegionInfo.getEncodedName(), Bytes.toString(familyName)));
406       // archive mob family store files
407       MobUtils.archiveMobStoreFiles(conf, fs, mobRegionInfo, mobFamilyDir, familyName);
408
409       if (!fs.delete(mobFamilyDir, true)) {
410         throw new IOException("Could not delete mob store files for family "
411             + Bytes.toString(familyName) + " from FileSystem region "
412             + mobRegionInfo.getRegionNameAsString() + "(" + mobRegionInfo.getEncodedName() + ")");
413       }
414     }
415   }
416
417   public void stop() {
418   }
419
420   public void logFileSystemState(Log log) throws IOException {
421     FSUtils.logFileSystemState(fs, rootdir, log);
422   }
423 }