View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.master;
20  
21  import java.io.IOException;
22
23  import org.apache.commons.logging.Log;
24  import org.apache.commons.logging.LogFactory;
25  import org.apache.hadoop.conf.Configuration;
26  import org.apache.hadoop.fs.FileSystem;
27  import org.apache.hadoop.fs.Path;
28  import org.apache.hadoop.fs.permission.FsPermission;
29  import org.apache.hadoop.hbase.ClusterId;
30  import org.apache.hadoop.hbase.HColumnDescriptor;
31  import org.apache.hadoop.hbase.HConstants;
32  import org.apache.hadoop.hbase.HRegionInfo;
33  import org.apache.hadoop.hbase.HTableDescriptor;
34  import org.apache.hadoop.hbase.TableDescriptor;
35  import org.apache.hadoop.hbase.TableName;
36  import org.apache.hadoop.hbase.backup.HFileArchiver;
37  import org.apache.hadoop.hbase.classification.InterfaceAudience;
38  import org.apache.hadoop.hbase.exceptions.DeserializationException;
39  import org.apache.hadoop.hbase.fs.HFileSystem;
40  import org.apache.hadoop.hbase.mob.MobConstants;
41  import org.apache.hadoop.hbase.mob.MobUtils;
42  import org.apache.hadoop.hbase.regionserver.HRegion;
43  import org.apache.hadoop.hbase.util.Bytes;
44  import org.apache.hadoop.hbase.util.FSTableDescriptors;
45  import org.apache.hadoop.hbase.util.FSUtils;
46  import org.apache.hadoop.ipc.RemoteException;
47
48  /**
49   * This class abstracts a bunch of operations the HMaster needs to interact with
50   * the underlying file system like creating the initial layout, checking file
51   * system status, etc.
52   */
53  @InterfaceAudience.Private
54  public class MasterFileSystem {
55    private static final Log LOG = LogFactory.getLog(MasterFileSystem.class);
56
57    // HBase configuration
58    private final Configuration conf;
59    // Persisted unique cluster ID
60    private ClusterId clusterId;
61    // Keep around for convenience.
62    private final FileSystem fs;
63    // root hbase directory on the FS
64    private final Path rootdir;
65    // hbase temp directory used for table construction and deletion
66    private final Path tempdir;
67
68    private final MasterServices services;
69
70    public MasterFileSystem(MasterServices services) throws IOException {
71      this.conf = services.getConfiguration();
72      this.services = services;
73      // Set filesystem to be that of this.rootdir else we get complaints about
74      // mismatched filesystems if hbase.rootdir is hdfs and fs.defaultFS is
75      // default localfs.  Presumption is that rootdir is fully-qualified before
76      // we get to here with appropriate fs scheme.
77      this.rootdir = FSUtils.getRootDir(conf);
78      this.tempdir = new Path(this.rootdir, HConstants.HBASE_TEMP_DIRECTORY);
79      // Cover both bases, the old way of setting default fs and the new.
80      // We're supposed to run on 0.20 and 0.21 anyways.
81      this.fs = this.rootdir.getFileSystem(conf);
82      FSUtils.setFsDefault(conf, new Path(this.fs.getUri()));
83      // make sure the fs has the same conf
84      fs.setConf(conf);
85      // setup the filesystem variable
86      createInitialFileSystemLayout();
87      HFileSystem.addLocationsOrderInterceptor(conf);
88    }
89
90    /**
91     * Create initial layout in filesystem.
92     * <ol>
93     * <li>Check if the meta region exists and is readable, if not create it.
94     * Create hbase.version and the hbase:meta directory if not one.
95     * </li>
96     * </ol>
97     * Idempotent.
98     */
99    private void createInitialFileSystemLayout() throws IOException {
100     // check if the root directory exists
101     checkRootDir(this.rootdir, conf, this.fs);
102
103     // check if temp directory exists and clean it
104     checkTempDir(this.tempdir, conf, this.fs);
105   }
106
107   public FileSystem getFileSystem() {
108     return this.fs;
109   }
110 
111   public Configuration getConfiguration() {
112     return this.conf;
113   }
114
115   /**
116    * @return HBase root dir.
117    */
118   public Path getRootDir() {
119     return this.rootdir;
120   }
121
122   /**
123    * @return HBase temp dir.
124    */
125   public Path getTempDir() {
126     return this.tempdir;
127   }
128
129   /**
130    * @return The unique identifier generated for this cluster
131    */
132   public ClusterId getClusterId() {
133     return clusterId;
134   }
135
136   /**
137    * Get the rootdir.  Make sure its wholesome and exists before returning.
138    * @param rd
139    * @param c
140    * @param fs
141    * @return hbase.rootdir (after checks for existence and bootstrapping if
142    * needed populating the directory with necessary bootup files).
143    * @throws IOException
144    */
145   private Path checkRootDir(final Path rd, final Configuration c, final FileSystem fs)
146       throws IOException {
147     // If FS is in safe mode wait till out of it.
148     FSUtils.waitOnSafeMode(c, c.getInt(HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000));
149
150     boolean isSecurityEnabled = "kerberos".equalsIgnoreCase(c.get("hbase.security.authentication"));
151     FsPermission rootDirPerms = new FsPermission(c.get("hbase.rootdir.perms", "700"));
152
153     // Filesystem is good. Go ahead and check for hbase.rootdir.
154     try {
155       if (!fs.exists(rd)) {
156         if (isSecurityEnabled) {
157           fs.mkdirs(rd, rootDirPerms);
158         } else {
159           fs.mkdirs(rd);
160         }
161         // DFS leaves safe mode with 0 DNs when there are 0 blocks.
162         // We used to handle this by checking the current DN count and waiting until
163         // it is nonzero. With security, the check for datanode count doesn't work --
164         // it is a privileged op. So instead we adopt the strategy of the jobtracker
165         // and simply retry file creation during bootstrap indefinitely. As soon as
166         // there is one datanode it will succeed. Permission problems should have
167         // already been caught by mkdirs above.
168         FSUtils.setVersion(fs, rd, c.getInt(HConstants.THREAD_WAKE_FREQUENCY,
169           10 * 1000), c.getInt(HConstants.VERSION_FILE_WRITE_ATTEMPTS,
170             HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS));
171       } else {
172         if (!fs.isDirectory(rd)) {
173           throw new IllegalArgumentException(rd.toString() + " is not a directory");
174         }
175         if (isSecurityEnabled && !rootDirPerms.equals(fs.getFileStatus(rd).getPermission())) {
176           // check whether the permission match
177           LOG.warn("Found rootdir permissions NOT matching expected \"hbase.rootdir.perms\" for "
178               + "rootdir=" + rd.toString() + " permissions=" + fs.getFileStatus(rd).getPermission()
179               + " and  \"hbase.rootdir.perms\" configured as "
180               + c.get("hbase.rootdir.perms", "700") + ". Automatically setting the permissions. You"
181               + " can change the permissions by setting \"hbase.rootdir.perms\" in hbase-site.xml "
182               + "and restarting the master");
183           fs.setPermission(rd, rootDirPerms);
184         }
185         // as above
186         FSUtils.checkVersion(fs, rd, true, c.getInt(HConstants.THREAD_WAKE_FREQUENCY,
187           10 * 1000), c.getInt(HConstants.VERSION_FILE_WRITE_ATTEMPTS,
188             HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS));
189       }
190     } catch (DeserializationException de) {
191       LOG.fatal("Please fix invalid configuration for " + HConstants.HBASE_DIR, de);
192       IOException ioe = new IOException();
193       ioe.initCause(de);
194       throw ioe;
195     } catch (IllegalArgumentException iae) {
196       LOG.fatal("Please fix invalid configuration for "
197         + HConstants.HBASE_DIR + " " + rd.toString(), iae);
198       throw iae;
199     }
200     // Make sure cluster ID exists
201     if (!FSUtils.checkClusterIdExists(fs, rd, c.getInt(
202         HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000))) {
203       FSUtils.setClusterId(fs, rd, new ClusterId(), c.getInt(HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000));
204     }
205     clusterId = FSUtils.getClusterId(fs, rd);
206 
207     // Make sure the meta region directory exists!
208     if (!FSUtils.metaRegionExists(fs, rd)) {
209       bootstrap(rd, c);
210     }
211
212     // Create tableinfo-s for hbase:meta if not already there.
213     // assume, created table descriptor is for enabling table
214     // meta table is a system table, so descriptors are predefined,
215     // we should get them from registry.
216     FSTableDescriptors fsd = new FSTableDescriptors(c, fs, rd);
217     fsd.createTableDescriptor(
218         new TableDescriptor(fsd.get(TableName.META_TABLE_NAME)));
219
220     return rd;
221   }
222
223   /**
224    * Make sure the hbase temp directory exists and is empty.
225    * NOTE that this method is only executed once just after the master becomes the active one.
226    */
227   private void checkTempDir(final Path tmpdir, final Configuration c, final FileSystem fs)
228       throws IOException {
229     // If the temp directory exists, clear the content (left over, from the previous run)
230     if (fs.exists(tmpdir)) {
231       // Archive table in temp, maybe left over from failed deletion,
232       // if not the cleaner will take care of them.
233       for (Path tabledir: FSUtils.getTableDirs(fs, tmpdir)) {
234         for (Path regiondir: FSUtils.getRegionDirs(fs, tabledir)) {
235           HFileArchiver.archiveRegion(fs, this.rootdir, tabledir, regiondir);
236         }
237       }
238       if (!fs.delete(tmpdir, true)) {
239         throw new IOException("Unable to clean the temp directory: " + tmpdir);
240       }
241     }
242
243     // Create the temp directory
244     if (!fs.mkdirs(tmpdir)) {
245       throw new IOException("HBase temp directory '" + tmpdir + "' creation failure.");
246     }
247   }
248
249   private static void bootstrap(final Path rd, final Configuration c)
250   throws IOException {
251     LOG.info("BOOTSTRAP: creating hbase:meta region");
252     try {
253       // Bootstrapping, make sure blockcache is off.  Else, one will be
254       // created here in bootstrap and it'll need to be cleaned up.  Better to
255       // not make it in first place.  Turn off block caching for bootstrap.
256       // Enable after.
257       HRegionInfo metaHRI = new HRegionInfo(HRegionInfo.FIRST_META_REGIONINFO);
258       HTableDescriptor metaDescriptor = new FSTableDescriptors(c).get(TableName.META_TABLE_NAME);
259       setInfoFamilyCachingForMeta(metaDescriptor, false);
260       HRegion meta = HRegion.createHRegion(metaHRI, rd, c, metaDescriptor, null);
261       setInfoFamilyCachingForMeta(metaDescriptor, true);
262       meta.close();
263     } catch (IOException e) {
264         e = e instanceof RemoteException ?
265                 ((RemoteException)e).unwrapRemoteException() : e;
266       LOG.error("bootstrap", e);
267       throw e;
268     }
269   }
270
271   /**
272    * Enable in memory caching for hbase:meta
273    */
274   public static void setInfoFamilyCachingForMeta(HTableDescriptor metaDescriptor, final boolean b) {
275     for (HColumnDescriptor hcd: metaDescriptor.getColumnFamilies()) {
276       if (Bytes.equals(hcd.getName(), HConstants.CATALOG_FAMILY)) {
277         hcd.setBlockCacheEnabled(b);
278         hcd.setInMemory(b);
279       }
280     }
281   }
282
283   public void deleteFamilyFromFS(HRegionInfo region, byte[] familyName, boolean hasMob)
284       throws IOException {
285     // archive family store files
286     Path tableDir = FSUtils.getTableDir(rootdir, region.getTable());
287     HFileArchiver.archiveFamily(fs, conf, region, tableDir, familyName);
288 
289     // delete the family folder
290     Path familyDir = new Path(tableDir,
291       new Path(region.getEncodedName(), Bytes.toString(familyName)));
292     if (fs.delete(familyDir, true) == false) {
293       if (fs.exists(familyDir)) {
294         throw new IOException("Could not delete family "
295             + Bytes.toString(familyName) + " from FileSystem for region "
296             + region.getRegionNameAsString() + "(" + region.getEncodedName()
297             + ")");
298       }
299     }
300
301     // archive and delete mob files
302     if (hasMob) {
303       Path mobTableDir =
304           FSUtils.getTableDir(new Path(getRootDir(), MobConstants.MOB_DIR_NAME), region.getTable());
305       HRegionInfo mobRegionInfo = MobUtils.getMobRegionInfo(region.getTable());
306       Path mobFamilyDir =
307           new Path(mobTableDir,
308               new Path(mobRegionInfo.getEncodedName(), Bytes.toString(familyName)));
309       // archive mob family store files
310       MobUtils.archiveMobStoreFiles(conf, fs, mobRegionInfo, mobFamilyDir, familyName);
311
312       if (!fs.delete(mobFamilyDir, true)) {
313         throw new IOException("Could not delete mob store files for family "
314             + Bytes.toString(familyName) + " from FileSystem region "
315             + mobRegionInfo.getRegionNameAsString() + "(" + mobRegionInfo.getEncodedName() + ")");
316       }
317     }
318   }
319
320   public void stop() {
321   }
322
323   public void logFileSystemState(Log log) throws IOException {
324     FSUtils.logFileSystemState(fs, rootdir, log);
325   }
326 }