View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.util;
20  
21  import java.io.IOException;
22  import java.util.NavigableSet;
23  
24  import org.apache.commons.logging.Log;
25  import org.apache.commons.logging.LogFactory;
26  import org.apache.hadoop.classification.InterfaceAudience;
27  import org.apache.hadoop.fs.FileStatus;
28  import org.apache.hadoop.fs.FileSystem;
29  import org.apache.hadoop.fs.Path;
30  import org.apache.hadoop.fs.PathFilter;
31  import org.apache.hadoop.hbase.HConstants;
32  import org.apache.hadoop.hbase.HRegionInfo;
33  import org.apache.hadoop.hbase.io.Reference;
34  import org.apache.hadoop.hbase.regionserver.HRegion;
35  import org.apache.hadoop.hbase.regionserver.wal.HLogUtil;
36  import org.apache.hadoop.hbase.util.FSUtils;
37  
38  /**
39   * Utility methods for interacting with the hbase.root file system.
40   */
41  @InterfaceAudience.Private
42  public final class FSVisitor {
43    private static final Log LOG = LogFactory.getLog(FSVisitor.class);
44  
45    public interface StoreFileVisitor {
46      void storeFile(final String region, final String family, final String hfileName)
47         throws IOException;
48    }
49  
50    public interface RecoveredEditsVisitor {
51      void recoveredEdits (final String region, final String logfile)
52        throws IOException;
53    }
54  
55    public interface LogFileVisitor {
56      void logFile (final String server, final String logfile)
57        throws IOException;
58    }
59  
60    private FSVisitor() {
61      // private constructor for utility class
62    }
63  
64    /**
65     * Iterate over the table store files
66     *
67     * @param fs {@link FileSystem}
68     * @param tableDir {@link Path} to the table directory
69     * @param visitor callback object to get the store files
70     * @throws IOException if an error occurred while scanning the directory
71     */
72    public static void visitTableStoreFiles(final FileSystem fs, final Path tableDir,
73        final StoreFileVisitor visitor) throws IOException {
74      FileStatus[] regions = FSUtils.listStatus(fs, tableDir, new FSUtils.RegionDirFilter(fs));
75      if (regions == null) {
76        LOG.info("No regions under directory:" + tableDir);
77        return;
78      }
79  
80      for (FileStatus region: regions) {
81        visitRegionStoreFiles(fs, region.getPath(), visitor);
82      }
83    }
84  
85    /**
86     * Iterate over the region store files
87     *
88     * @param fs {@link FileSystem}
89     * @param regionDir {@link Path} to the region directory
90     * @param visitor callback object to get the store files
91     * @throws IOException if an error occurred while scanning the directory
92     */
93    public static void visitRegionStoreFiles(final FileSystem fs, final Path regionDir,
94        final StoreFileVisitor visitor) throws IOException {
95      FileStatus[] families = FSUtils.listStatus(fs, regionDir, new FSUtils.FamilyDirFilter(fs));
96      if (families == null) {
97        LOG.info("No families under region directory:" + regionDir);
98        return;
99      }
100 
101     PathFilter fileFilter = new FSUtils.FileFilter(fs);
102     for (FileStatus family: families) {
103       Path familyDir = family.getPath();
104       String familyName = familyDir.getName();
105 
106       // get all the storeFiles in the family
107       FileStatus[] storeFiles = FSUtils.listStatus(fs, familyDir, fileFilter);
108       if (storeFiles == null) {
109         LOG.debug("No hfiles found for family: " + familyDir + ", skipping.");
110         continue;
111       }
112 
113       for (FileStatus hfile: storeFiles) {
114         Path hfilePath = hfile.getPath();
115         visitor.storeFile(regionDir.getName(), familyName, hfilePath.getName());
116       }
117     }
118   }
119 
120   /**
121    * Iterate over each region in the table and inform about recovered.edits
122    *
123    * @param fs {@link FileSystem}
124    * @param tableDir {@link Path} to the table directory
125    * @param visitor callback object to get the recovered.edits files
126    * @throws IOException if an error occurred while scanning the directory
127    */
128   public static void visitTableRecoveredEdits(final FileSystem fs, final Path tableDir,
129       final FSVisitor.RecoveredEditsVisitor visitor) throws IOException {
130     FileStatus[] regions = FSUtils.listStatus(fs, tableDir, new FSUtils.RegionDirFilter(fs));
131     if (regions == null) {
132       LOG.info("No regions under directory:" + tableDir);
133       return;
134     }
135 
136     for (FileStatus region: regions) {
137       visitRegionRecoveredEdits(fs, region.getPath(), visitor);
138     }
139   }
140 
141   /**
142    * Iterate over recovered.edits of the specified region
143    *
144    * @param fs {@link FileSystem}
145    * @param regionDir {@link Path} to the Region directory
146    * @param visitor callback object to get the recovered.edits files
147    * @throws IOException if an error occurred while scanning the directory
148    */
149   public static void visitRegionRecoveredEdits(final FileSystem fs, final Path regionDir,
150       final FSVisitor.RecoveredEditsVisitor visitor) throws IOException {
151     NavigableSet<Path> files = HLogUtil.getSplitEditFilesSorted(fs, regionDir);
152     if (files == null || files.size() == 0) return;
153 
154     for (Path source: files) {
155       // check to see if the file is zero length, in which case we can skip it
156       FileStatus stat = fs.getFileStatus(source);
157       if (stat.getLen() <= 0) continue;
158 
159       visitor.recoveredEdits(regionDir.getName(), source.getName());
160     }
161   }
162 
163   /**
164    * Iterate over hbase log files
165    *
166    * @param fs {@link FileSystem}
167    * @param rootDir {@link Path} to the HBase root folder
168    * @param visitor callback object to get the log files
169    * @throws IOException if an error occurred while scanning the directory
170    */
171   public static void visitLogFiles(final FileSystem fs, final Path rootDir,
172       final LogFileVisitor visitor) throws IOException {
173     Path logsDir = new Path(rootDir, HConstants.HREGION_LOGDIR_NAME);
174     FileStatus[] logServerDirs = FSUtils.listStatus(fs, logsDir);
175     if (logServerDirs == null) {
176       LOG.info("No logs under directory:" + logsDir);
177       return;
178     }
179 
180     for (FileStatus serverLogs: logServerDirs) {
181       String serverName = serverLogs.getPath().getName();
182 
183       FileStatus[] hlogs = FSUtils.listStatus(fs, serverLogs.getPath());
184       if (hlogs == null) {
185         LOG.debug("No hfiles found for server: " + serverName + ", skipping.");
186         continue;
187       }
188 
189       for (FileStatus hlogRef: hlogs) {
190         visitor.logFile(serverName, hlogRef.getPath().getName());
191       }
192     }
193   }
194 }