View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.util;
20  
21  import java.io.IOException;
22  import java.util.NavigableSet;
23  
24  import org.apache.commons.logging.Log;
25  import org.apache.commons.logging.LogFactory;
26  import org.apache.hadoop.classification.InterfaceAudience;
27  import org.apache.hadoop.fs.FileStatus;
28  import org.apache.hadoop.fs.FileSystem;
29  import org.apache.hadoop.fs.Path;
30  import org.apache.hadoop.fs.PathFilter;
31  import org.apache.hadoop.hbase.HConstants;
32  import org.apache.hadoop.hbase.regionserver.wal.HLogUtil;
33  
34  /**
35   * Utility methods for interacting with the hbase.root file system.
36   */
37  @InterfaceAudience.Private
38  public final class FSVisitor {
39    private static final Log LOG = LogFactory.getLog(FSVisitor.class);
40  
41    public interface RegionVisitor {
42      void region(final String region) throws IOException;
43    }
44  
45    public interface StoreFileVisitor {
46      void storeFile(final String region, final String family, final String hfileName)
47         throws IOException;
48    }
49  
50    public interface RecoveredEditsVisitor {
51      void recoveredEdits (final String region, final String logfile)
52        throws IOException;
53    }
54  
55    public interface LogFileVisitor {
56      void logFile (final String server, final String logfile)
57        throws IOException;
58    }
59  
60    private FSVisitor() {
61      // private constructor for utility class
62    }
63  
64    /**
65     * Iterate over the table store files
66     *
67     * @param fs {@link FileSystem}
68     * @param tableDir {@link Path} to the table directory
69     * @param visitor callback object to get the store files
70     * @throws IOException if an error occurred while scanning the directory
71     */
72    public static void visitRegions(final FileSystem fs, final Path tableDir,
73        final RegionVisitor visitor) throws IOException {
74      FileStatus[] regions = FSUtils.listStatus(fs, tableDir, new FSUtils.RegionDirFilter(fs));
75      if (regions == null) {
76        LOG.info("No regions under directory:" + tableDir);
77        return;
78      }
79  
80      for (FileStatus region: regions) {
81        visitor.region(region.getPath().getName());
82      }
83    }
84  
85    /**
86     * Iterate over the table store files
87     *
88     * @param fs {@link FileSystem}
89     * @param tableDir {@link Path} to the table directory
90     * @param visitor callback object to get the store files
91     * @throws IOException if an error occurred while scanning the directory
92     */
93    public static void visitTableStoreFiles(final FileSystem fs, final Path tableDir,
94        final StoreFileVisitor visitor) throws IOException {
95      FileStatus[] regions = FSUtils.listStatus(fs, tableDir, new FSUtils.RegionDirFilter(fs));
96      if (regions == null) {
97        LOG.info("No regions under directory:" + tableDir);
98        return;
99      }
100 
101     for (FileStatus region: regions) {
102       visitRegionStoreFiles(fs, region.getPath(), visitor);
103     }
104   }
105 
106   /**
107    * Iterate over the region store files
108    *
109    * @param fs {@link FileSystem}
110    * @param regionDir {@link Path} to the region directory
111    * @param visitor callback object to get the store files
112    * @throws IOException if an error occurred while scanning the directory
113    */
114   public static void visitRegionStoreFiles(final FileSystem fs, final Path regionDir,
115       final StoreFileVisitor visitor) throws IOException {
116     FileStatus[] families = FSUtils.listStatus(fs, regionDir, new FSUtils.FamilyDirFilter(fs));
117     if (families == null) {
118       LOG.info("No families under region directory:" + regionDir);
119       return;
120     }
121 
122     PathFilter fileFilter = new FSUtils.FileFilter(fs);
123     for (FileStatus family: families) {
124       Path familyDir = family.getPath();
125       String familyName = familyDir.getName();
126 
127       // get all the storeFiles in the family
128       FileStatus[] storeFiles = FSUtils.listStatus(fs, familyDir, fileFilter);
129       if (storeFiles == null) {
130         LOG.debug("No hfiles found for family: " + familyDir + ", skipping.");
131         continue;
132       }
133 
134       for (FileStatus hfile: storeFiles) {
135         Path hfilePath = hfile.getPath();
136         visitor.storeFile(regionDir.getName(), familyName, hfilePath.getName());
137       }
138     }
139   }
140 
141   /**
142    * Iterate over each region in the table and inform about recovered.edits
143    *
144    * @param fs {@link FileSystem}
145    * @param tableDir {@link Path} to the table directory
146    * @param visitor callback object to get the recovered.edits files
147    * @throws IOException if an error occurred while scanning the directory
148    */
149   public static void visitTableRecoveredEdits(final FileSystem fs, final Path tableDir,
150       final FSVisitor.RecoveredEditsVisitor visitor) throws IOException {
151     FileStatus[] regions = FSUtils.listStatus(fs, tableDir, new FSUtils.RegionDirFilter(fs));
152     if (regions == null) {
153       LOG.info("No recoveredEdits regions under directory:" + tableDir);
154       return;
155     }
156 
157     for (FileStatus region: regions) {
158       visitRegionRecoveredEdits(fs, region.getPath(), visitor);
159     }
160   }
161 
162   /**
163    * Iterate over recovered.edits of the specified region
164    *
165    * @param fs {@link FileSystem}
166    * @param regionDir {@link Path} to the Region directory
167    * @param visitor callback object to get the recovered.edits files
168    * @throws IOException if an error occurred while scanning the directory
169    */
170   public static void visitRegionRecoveredEdits(final FileSystem fs, final Path regionDir,
171       final FSVisitor.RecoveredEditsVisitor visitor) throws IOException {
172     NavigableSet<Path> files = HLogUtil.getSplitEditFilesSorted(fs, regionDir);
173     if (files == null || files.size() == 0) return;
174 
175     for (Path source: files) {
176       // check to see if the file is zero length, in which case we can skip it
177       FileStatus stat = fs.getFileStatus(source);
178       if (stat.getLen() <= 0) continue;
179 
180       visitor.recoveredEdits(regionDir.getName(), source.getName());
181     }
182   }
183 
184   /**
185    * Iterate over hbase log files
186    *
187    * @param fs {@link FileSystem}
188    * @param rootDir {@link Path} to the HBase root folder
189    * @param visitor callback object to get the log files
190    * @throws IOException if an error occurred while scanning the directory
191    */
192   public static void visitLogFiles(final FileSystem fs, final Path rootDir,
193       final LogFileVisitor visitor) throws IOException {
194     Path logsDir = new Path(rootDir, HConstants.HREGION_LOGDIR_NAME);
195     FileStatus[] logServerDirs = FSUtils.listStatus(fs, logsDir);
196     if (logServerDirs == null) {
197       LOG.info("No logs under directory:" + logsDir);
198       return;
199     }
200 
201     for (FileStatus serverLogs: logServerDirs) {
202       String serverName = serverLogs.getPath().getName();
203 
204       FileStatus[] hlogs = FSUtils.listStatus(fs, serverLogs.getPath());
205       if (hlogs == null) {
206         LOG.debug("No hfiles found for server: " + serverName + ", skipping.");
207         continue;
208       }
209 
210       for (FileStatus hlogRef: hlogs) {
211         visitor.logFile(serverName, hlogRef.getPath().getName());
212       }
213     }
214   }
215 }