View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.snapshot;
20  
21  import java.io.IOException;
22  import java.io.FileNotFoundException;
23  import java.util.HashSet;
24  import java.util.TreeMap;
25  import java.util.LinkedList;
26  import java.util.List;
27  import java.util.Map;
28  import java.util.Set;
29  
30  import org.apache.commons.logging.Log;
31  import org.apache.commons.logging.LogFactory;
32  
33  import org.apache.hadoop.conf.Configuration;
34  import org.apache.hadoop.classification.InterfaceAudience;
35  import org.apache.hadoop.fs.Path;
36  import org.apache.hadoop.fs.FileSystem;
37  import org.apache.hadoop.fs.FileStatus;
38  
39  import org.apache.hadoop.hbase.HConstants;
40  import org.apache.hadoop.hbase.io.Reference;
41  import org.apache.hadoop.hbase.io.HFileLink;
42  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
43  import org.apache.hadoop.hbase.regionserver.wal.HLog;
44  import org.apache.hadoop.hbase.util.FSUtils;
45  import org.apache.hadoop.hbase.util.FSVisitor;
46  
47  /**
48   * Utility methods for interacting with the snapshot referenced files.
49   */
50  @InterfaceAudience.Private
51  public final class SnapshotReferenceUtil {
52    public interface FileVisitor extends FSVisitor.StoreFileVisitor,
53      FSVisitor.RecoveredEditsVisitor, FSVisitor.LogFileVisitor {
54    }
55  
56    private SnapshotReferenceUtil() {
57      // private constructor for utility class
58    }
59  
60    /**
61     * Get log directory for a server in a snapshot.
62     *
63     * @param snapshotDir directory where the specific snapshot is stored
64     * @param serverName name of the parent regionserver for the log files
65     * @return path to the log home directory for the archive files.
66     */
67    public static Path getLogsDir(Path snapshotDir, String serverName) {
68      return new Path(snapshotDir, HLog.getHLogDirectoryName(serverName));
69    }
70  
71    /**
72     * Get the snapshotted recovered.edits dir for the specified region.
73     *
74     * @param snapshotDir directory where the specific snapshot is stored
75     * @param regionName name of the region
76     * @return path to the recovered.edits directory for the specified region files.
77     */
78    public static Path getRecoveredEditsDir(Path snapshotDir, String regionName) {
79      return HLog.getRegionDirRecoveredEditsDir(new Path(snapshotDir, regionName));
80    }
81  
82    /**
83     * Get the snapshot recovered.edits file
84     *
85     * @param snapshotDir directory where the specific snapshot is stored
86     * @param regionName name of the region
87     * @param logfile name of the edit file
88     * @return full path of the log file for the specified region files.
89     */
90    public static Path getRecoveredEdits(Path snapshotDir, String regionName, String logfile) {
91      return new Path(getRecoveredEditsDir(snapshotDir, regionName), logfile);
92    }
93  
94    /**
95     * Iterate over the snapshot store files, restored.edits and logs
96     *
97     * @param fs {@link FileSystem}
98     * @param snapshotDir {@link Path} to the Snapshot directory
99     * @param visitor callback object to get the referenced files
100    * @throws IOException if an error occurred while scanning the directory
101    */
102   public static void visitReferencedFiles(final FileSystem fs, final Path snapshotDir,
103       final FileVisitor visitor) throws IOException {
104     visitTableStoreFiles(fs, snapshotDir, visitor);
105     visitRecoveredEdits(fs, snapshotDir, visitor);
106     visitLogFiles(fs, snapshotDir, visitor);
107   }
108 
109   /**
110    * Iterate over the snapshot store files
111    *
112    * @param fs {@link FileSystem}
113    * @param snapshotDir {@link Path} to the Snapshot directory
114    * @param visitor callback object to get the store files
115    * @throws IOException if an error occurred while scanning the directory
116    */
117   public static void visitTableStoreFiles(final FileSystem fs, final Path snapshotDir,
118       final FSVisitor.StoreFileVisitor visitor) throws IOException {
119     FSVisitor.visitTableStoreFiles(fs, snapshotDir, visitor);
120   }
121 
122   /**
123    * Iterate over the snapshot store files in the specified region
124    *
125    * @param fs {@link FileSystem}
126    * @param regionDir {@link Path} to the Snapshot region directory
127    * @param visitor callback object to get the store files
128    * @throws IOException if an error occurred while scanning the directory
129    */
130   public static void visitRegionStoreFiles(final FileSystem fs, final Path regionDir,
131       final FSVisitor.StoreFileVisitor visitor) throws IOException {
132     FSVisitor.visitRegionStoreFiles(fs, regionDir, visitor);
133   }
134 
135   /**
136    * Iterate over the snapshot recovered.edits
137    *
138    * @param fs {@link FileSystem}
139    * @param snapshotDir {@link Path} to the Snapshot directory
140    * @param visitor callback object to get the recovered.edits files
141    * @throws IOException if an error occurred while scanning the directory
142    */
143   public static void visitRecoveredEdits(final FileSystem fs, final Path snapshotDir,
144       final FSVisitor.RecoveredEditsVisitor visitor) throws IOException {
145     FSVisitor.visitTableRecoveredEdits(fs, snapshotDir, visitor);
146   }
147 
148   /**
149    * Iterate over the snapshot log files
150    *
151    * @param fs {@link FileSystem}
152    * @param snapshotDir {@link Path} to the Snapshot directory
153    * @param visitor callback object to get the log files
154    * @throws IOException if an error occurred while scanning the directory
155    */
156   public static void visitLogFiles(final FileSystem fs, final Path snapshotDir,
157       final FSVisitor.LogFileVisitor visitor) throws IOException {
158     FSVisitor.visitLogFiles(fs, snapshotDir, visitor);
159   }
160 
161   /**
162    * Verify the validity of the snapshot
163    *
164    * @param conf The current {@link Configuration} instance.
165    * @param fs {@link FileSystem}
166    * @param snapshotDir {@link Path} to the Snapshot directory of the snapshot to verify
167    * @param snapshotDesc the {@link SnapshotDescription} of the snapshot to verify
168    * @throws CorruptedSnapshotException if the snapshot is corrupted
169    * @throws IOException if an error occurred while scanning the directory
170    */
171   public static void verifySnapshot(final Configuration conf, final FileSystem fs,
172       final Path snapshotDir, final SnapshotDescription snapshotDesc) throws IOException {
173     final String table = snapshotDesc.getTable();
174     visitTableStoreFiles(fs, snapshotDir, new FSVisitor.StoreFileVisitor() {
175       public void storeFile (final String region, final String family, final String hfile)
176           throws IOException {
177         HFileLink link = HFileLink.create(conf, table, region, family, hfile);
178         try {
179           link.getFileStatus(fs);
180         } catch (FileNotFoundException e) {
181           throw new CorruptedSnapshotException("Corrupted snapshot '" + snapshotDesc + "'", e);
182         }
183       }
184     });
185   }
186 
187   /**
188    * Returns the set of region names available in the snapshot.
189    *
190    * @param fs {@link FileSystem}
191    * @param snapshotDir {@link Path} to the Snapshot directory
192    * @throws IOException if an error occurred while scanning the directory
193    * @return the set of the regions contained in the snapshot
194    */
195   public static Set<String> getSnapshotRegionNames(final FileSystem fs, final Path snapshotDir)
196       throws IOException {
197     FileStatus[] regionDirs = FSUtils.listStatus(fs, snapshotDir, new FSUtils.RegionDirFilter(fs));
198     if (regionDirs == null) return null;
199 
200     Set<String> regions = new HashSet<String>();
201     for (FileStatus regionDir: regionDirs) {
202       regions.add(regionDir.getPath().getName());
203     }
204     return regions;
205   }
206 
207   /**
208    * Get the list of hfiles for the specified snapshot region.
209    * NOTE: The current implementation keeps one empty file per HFile in the region.
210    * The file name matches the one in the original table, and by reconstructing
211    * the path you can quickly jump to the referenced file.
212    *
213    * @param fs {@link FileSystem}
214    * @param snapshotRegionDir {@link Path} to the Snapshot region directory
215    * @return Map of hfiles per family, the key is the family name and values are hfile names
216    * @throws IOException if an error occurred while scanning the directory
217    */
218   public static Map<String, List<String>> getRegionHFileReferences(final FileSystem fs,
219       final Path snapshotRegionDir) throws IOException {
220     final Map<String, List<String>> familyFiles = new TreeMap<String, List<String>>();
221 
222     visitRegionStoreFiles(fs, snapshotRegionDir,
223       new FSVisitor.StoreFileVisitor() {
224         public void storeFile (final String region, final String family, final String hfile)
225             throws IOException {
226           List<String> hfiles = familyFiles.get(family);
227           if (hfiles == null) {
228             hfiles = new LinkedList<String>();
229             familyFiles.put(family, hfiles);
230           }
231           hfiles.add(hfile);
232         }
233     });
234 
235     return familyFiles;
236   }
237 
238   /**
239    * Returns the store file names in the snapshot.
240    *
241    * @param fs {@link FileSystem}
242    * @param snapshotDir {@link Path} to the Snapshot directory
243    * @throws IOException if an error occurred while scanning the directory
244    * @return the names of hfiles in the specified snaphot
245    */
246   public static Set<String> getHFileNames(final FileSystem fs, final Path snapshotDir)
247       throws IOException {
248     final Set<String> names = new HashSet<String>();
249     visitTableStoreFiles(fs, snapshotDir, new FSVisitor.StoreFileVisitor() {
250       public void storeFile (final String region, final String family, final String hfile)
251           throws IOException {
252         if (HFileLink.isHFileLink(hfile)) {
253           names.add(HFileLink.getReferencedHFileName(hfile));
254         } else {
255           names.add(hfile);
256         }
257       }
258     });
259     return names;
260   }
261 
262   /**
263    * Returns the log file names available in the snapshot.
264    *
265    * @param fs {@link FileSystem}
266    * @param snapshotDir {@link Path} to the Snapshot directory
267    * @throws IOException if an error occurred while scanning the directory
268    * @return the names of hlogs in the specified snaphot
269    */
270   public static Set<String> getHLogNames(final FileSystem fs, final Path snapshotDir)
271       throws IOException {
272     final Set<String> names = new HashSet<String>();
273     visitLogFiles(fs, snapshotDir, new FSVisitor.LogFileVisitor() {
274       public void logFile (final String server, final String logfile) throws IOException {
275         names.add(logfile);
276       }
277     });
278     return names;
279   }
280 }