View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.snapshot;
20  
21  import java.io.IOException;
22  import java.io.FileNotFoundException;
23  import java.util.HashSet;
24  import java.util.LinkedList;
25  import java.util.List;
26  import java.util.Map;
27  import java.util.Set;
28  import java.util.TreeMap;
29  
30  import org.apache.hadoop.classification.InterfaceAudience;
31  import org.apache.hadoop.conf.Configuration;
32  import org.apache.hadoop.fs.FileStatus;
33  import org.apache.hadoop.fs.FileSystem;
34  import org.apache.hadoop.fs.Path;
35  import org.apache.hadoop.hbase.TableName;
36  import org.apache.hadoop.hbase.io.HFileLink;
37  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
38  import org.apache.hadoop.hbase.regionserver.wal.HLogUtil;
39  import org.apache.hadoop.hbase.util.FSUtils;
40  import org.apache.hadoop.hbase.util.FSVisitor;
41  
42  /**
43   * Utility methods for interacting with the snapshot referenced files.
44   */
45  @InterfaceAudience.Private
46  public final class SnapshotReferenceUtil {
47    public interface FileVisitor extends FSVisitor.StoreFileVisitor,
48      FSVisitor.RecoveredEditsVisitor, FSVisitor.LogFileVisitor {
49    }
50  
51    private SnapshotReferenceUtil() {
52      // private constructor for utility class
53    }
54  
55    /**
56     * Get log directory for a server in a snapshot.
57     *
58     * @param snapshotDir directory where the specific snapshot is stored
59     * @param serverName name of the parent regionserver for the log files
60     * @return path to the log home directory for the archive files.
61     */
62    public static Path getLogsDir(Path snapshotDir, String serverName) {
63      return new Path(snapshotDir, HLogUtil.getHLogDirectoryName(serverName));
64    }
65  
66    /**
67     * Get the snapshotted recovered.edits dir for the specified region.
68     *
69     * @param snapshotDir directory where the specific snapshot is stored
70     * @param regionName name of the region
71     * @return path to the recovered.edits directory for the specified region files.
72     */
73    public static Path getRecoveredEditsDir(Path snapshotDir, String regionName) {
74      return HLogUtil.getRegionDirRecoveredEditsDir(new Path(snapshotDir, regionName));
75    }
76  
77    /**
78     * Get the snapshot recovered.edits file
79     *
80     * @param snapshotDir directory where the specific snapshot is stored
81     * @param regionName name of the region
82     * @param logfile name of the edit file
83     * @return full path of the log file for the specified region files.
84     */
85    public static Path getRecoveredEdits(Path snapshotDir, String regionName, String logfile) {
86      return new Path(getRecoveredEditsDir(snapshotDir, regionName), logfile);
87    }
88  
89    /**
90     * Iterate over the snapshot store files, restored.edits and logs
91     *
92     * @param fs {@link FileSystem}
93     * @param snapshotDir {@link Path} to the Snapshot directory
94     * @param visitor callback object to get the referenced files
95     * @throws IOException if an error occurred while scanning the directory
96     */
97    public static void visitReferencedFiles(final FileSystem fs, final Path snapshotDir,
98        final FileVisitor visitor) throws IOException {
99      visitTableStoreFiles(fs, snapshotDir, visitor);
100     visitRecoveredEdits(fs, snapshotDir, visitor);
101     visitLogFiles(fs, snapshotDir, visitor);
102   }
103 
104   /**
105    * Iterate over the snapshot store files
106    *
107    * @param fs {@link FileSystem}
108    * @param snapshotDir {@link Path} to the Snapshot directory
109    * @param visitor callback object to get the store files
110    * @throws IOException if an error occurred while scanning the directory
111    */
112   public static void visitTableStoreFiles(final FileSystem fs, final Path snapshotDir,
113       final FSVisitor.StoreFileVisitor visitor) throws IOException {
114     FSVisitor.visitTableStoreFiles(fs, snapshotDir, visitor);
115   }
116 
117   /**
118    * Iterate over the snapshot store files in the specified region
119    *
120    * @param fs {@link FileSystem}
121    * @param regionDir {@link Path} to the Snapshot region directory
122    * @param visitor callback object to get the store files
123    * @throws IOException if an error occurred while scanning the directory
124    */
125   public static void visitRegionStoreFiles(final FileSystem fs, final Path regionDir,
126       final FSVisitor.StoreFileVisitor visitor) throws IOException {
127     FSVisitor.visitRegionStoreFiles(fs, regionDir, visitor);
128   }
129 
130   /**
131    * Iterate over the snapshot recovered.edits
132    *
133    * @param fs {@link FileSystem}
134    * @param snapshotDir {@link Path} to the Snapshot directory
135    * @param visitor callback object to get the recovered.edits files
136    * @throws IOException if an error occurred while scanning the directory
137    */
138   public static void visitRecoveredEdits(final FileSystem fs, final Path snapshotDir,
139       final FSVisitor.RecoveredEditsVisitor visitor) throws IOException {
140     FSVisitor.visitTableRecoveredEdits(fs, snapshotDir, visitor);
141   }
142 
143   /**
144    * Iterate over the snapshot log files
145    *
146    * @param fs {@link FileSystem}
147    * @param snapshotDir {@link Path} to the Snapshot directory
148    * @param visitor callback object to get the log files
149    * @throws IOException if an error occurred while scanning the directory
150    */
151   public static void visitLogFiles(final FileSystem fs, final Path snapshotDir,
152       final FSVisitor.LogFileVisitor visitor) throws IOException {
153     FSVisitor.visitLogFiles(fs, snapshotDir, visitor);
154   }
155 
156   /**
157    * Verify the validity of the snapshot
158    *
159    * @param conf The current {@link Configuration} instance.
160    * @param fs {@link FileSystem}
161    * @param snapshotDir {@link Path} to the Snapshot directory of the snapshot to verify
162    * @param snapshotDesc the {@link SnapshotDescription} of the snapshot to verify
163    * @throws CorruptedSnapshotException if the snapshot is corrupted
164    * @throws IOException if an error occurred while scanning the directory
165    */
166   public static void verifySnapshot(final Configuration conf, final FileSystem fs,
167       final Path snapshotDir, final SnapshotDescription snapshotDesc) throws IOException {
168     final TableName table = TableName.valueOf(snapshotDesc.getTable());
169     visitTableStoreFiles(fs, snapshotDir, new FSVisitor.StoreFileVisitor() {
170       public void storeFile (final String region, final String family, final String hfile)
171           throws IOException {
172         HFileLink link = HFileLink.create(conf, table, region, family, hfile);
173         try {
174           link.getFileStatus(fs);
175         } catch (FileNotFoundException e) {
176           throw new CorruptedSnapshotException("Corrupted snapshot '" + snapshotDesc + "'", e);
177         }
178       }
179     });
180   }
181 
182   /**
183    * Returns the set of region names available in the snapshot.
184    *
185    * @param fs {@link FileSystem}
186    * @param snapshotDir {@link Path} to the Snapshot directory
187    * @throws IOException if an error occurred while scanning the directory
188    * @return the set of the regions contained in the snapshot
189    */
190   public static Set<String> getSnapshotRegionNames(final FileSystem fs, final Path snapshotDir)
191       throws IOException {
192     FileStatus[] regionDirs = FSUtils.listStatus(fs, snapshotDir, new FSUtils.RegionDirFilter(fs));
193     if (regionDirs == null) return null;
194 
195     Set<String> regions = new HashSet<String>();
196     for (FileStatus regionDir: regionDirs) {
197       regions.add(regionDir.getPath().getName());
198     }
199     return regions;
200   }
201 
202   /**
203    * Get the list of hfiles for the specified snapshot region.
204    * NOTE: The current implementation keeps one empty file per HFile in the region.
205    * The file name matches the one in the original table, and by reconstructing
206    * the path you can quickly jump to the referenced file.
207    *
208    * @param fs {@link FileSystem}
209    * @param snapshotRegionDir {@link Path} to the Snapshot region directory
210    * @return Map of hfiles per family, the key is the family name and values are hfile names
211    * @throws IOException if an error occurred while scanning the directory
212    */
213   public static Map<String, List<String>> getRegionHFileReferences(final FileSystem fs,
214       final Path snapshotRegionDir) throws IOException {
215     final Map<String, List<String>> familyFiles = new TreeMap<String, List<String>>();
216 
217     visitRegionStoreFiles(fs, snapshotRegionDir,
218       new FSVisitor.StoreFileVisitor() {
219         public void storeFile (final String region, final String family, final String hfile)
220             throws IOException {
221           List<String> hfiles = familyFiles.get(family);
222           if (hfiles == null) {
223             hfiles = new LinkedList<String>();
224             familyFiles.put(family, hfiles);
225           }
226           hfiles.add(hfile);
227         }
228     });
229 
230     return familyFiles;
231   }
232 
233   /**
234    * Returns the store file names in the snapshot.
235    *
236    * @param fs {@link FileSystem}
237    * @param snapshotDir {@link Path} to the Snapshot directory
238    * @throws IOException if an error occurred while scanning the directory
239    * @return the names of hfiles in the specified snaphot
240    */
241   public static Set<String> getHFileNames(final FileSystem fs, final Path snapshotDir)
242       throws IOException {
243     final Set<String> names = new HashSet<String>();
244     visitTableStoreFiles(fs, snapshotDir, new FSVisitor.StoreFileVisitor() {
245       public void storeFile (final String region, final String family, final String hfile)
246           throws IOException {
247         if (HFileLink.isHFileLink(hfile)) {
248           names.add(HFileLink.getReferencedHFileName(hfile));
249         } else {
250           names.add(hfile);
251         }
252       }
253     });
254     return names;
255   }
256 
257   /**
258    * Returns the log file names available in the snapshot.
259    *
260    * @param fs {@link FileSystem}
261    * @param snapshotDir {@link Path} to the Snapshot directory
262    * @throws IOException if an error occurred while scanning the directory
263    * @return the names of hlogs in the specified snaphot
264    */
265   public static Set<String> getHLogNames(final FileSystem fs, final Path snapshotDir)
266       throws IOException {
267     final Set<String> names = new HashSet<String>();
268     visitLogFiles(fs, snapshotDir, new FSVisitor.LogFileVisitor() {
269       public void logFile (final String server, final String logfile) throws IOException {
270         names.add(logfile);
271       }
272     });
273     return names;
274   }
275 }