View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.snapshot;
20  
21  import java.io.IOException;
22  import java.io.FileNotFoundException;
23  import java.net.URI;
24  import java.text.SimpleDateFormat;
25  import java.util.ArrayList;
26  import java.util.Date;
27  import java.util.List;
28  import java.util.concurrent.atomic.AtomicInteger;
29  import java.util.concurrent.atomic.AtomicLong;
30  
31  import org.apache.commons.logging.Log;
32  import org.apache.commons.logging.LogFactory;
33  
34  import org.apache.hadoop.fs.Path;
35  import org.apache.hadoop.fs.FileStatus;
36  import org.apache.hadoop.fs.FileSystem;
37  import org.apache.hadoop.classification.InterfaceAudience;
38  import org.apache.hadoop.classification.InterfaceStability;
39  import org.apache.hadoop.conf.Configured;
40  import org.apache.hadoop.hbase.HRegionInfo;
41  import org.apache.hadoop.hbase.TableName;
42  import org.apache.hadoop.util.StringUtils;
43  import org.apache.hadoop.util.Tool;
44  import org.apache.hadoop.util.ToolRunner;
45  
46  import org.apache.hadoop.conf.Configuration;
47  import org.apache.hadoop.hbase.HBaseConfiguration;
48  import org.apache.hadoop.hbase.HTableDescriptor;
49  import org.apache.hadoop.hbase.io.HFileLink;
50  import org.apache.hadoop.hbase.io.HLogLink;
51  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
52  import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
53  import org.apache.hadoop.hbase.util.FSUtils;
54  
55  /**
56   * Tool for dumping snapshot information.
57   * <ol>
58   * <li> Table Descriptor
59   * <li> Snapshot creation time, type, format version, ...
60   * <li> List of hfiles and hlogs
61   * <li> Stats about hfiles and logs sizes, percentage of shared with the source table, ...
62   * </ol>
63   */
64  @InterfaceAudience.Public
65  @InterfaceStability.Evolving
66  public final class SnapshotInfo extends Configured implements Tool {
67    private static final Log LOG = LogFactory.getLog(SnapshotInfo.class);
68  
69    /**
70     * Statistics about the snapshot
71     * <ol>
72     * <li> How many store files and logs are in the archive
73     * <li> How many store files and logs are shared with the table
74     * <li> Total store files and logs size and shared amount
75     * </ol>
76     */
77    public static class SnapshotStats {
78      /** Information about the file referenced by the snapshot */
79      static class FileInfo {
80        private final boolean inArchive;
81        private final long size;
82  
83        FileInfo(final boolean inArchive, final long size) {
84          this.inArchive = inArchive;
85          this.size = size;
86        }
87  
88        /** @return true if the file is in the archive */
89        public boolean inArchive() {
90          return this.inArchive;
91        }
92  
93        /** @return true if the file is missing */
94        public boolean isMissing() {
95          return this.size < 0;
96        }
97  
98        /** @return the file size */
99        public long getSize() {
100         return this.size;
101       }
102     }
103 
104     private AtomicInteger hfileArchiveCount = new AtomicInteger();
105     private AtomicInteger hfilesMissing = new AtomicInteger();
106     private AtomicInteger hfilesCount = new AtomicInteger();
107     private AtomicInteger logsMissing = new AtomicInteger();
108     private AtomicInteger logsCount = new AtomicInteger();
109     private AtomicLong hfileArchiveSize = new AtomicLong();
110     private AtomicLong hfileSize = new AtomicLong();
111     private AtomicLong logSize = new AtomicLong();
112 
113     private final SnapshotDescription snapshot;
114     private final TableName snapshotTable;
115     private final Configuration conf;
116     private final FileSystem fs;
117 
118     SnapshotStats(final Configuration conf, final FileSystem fs, final SnapshotDescription snapshot)
119     {
120       this.snapshot = snapshot;
121       this.snapshotTable = TableName.valueOf(snapshot.getTable());
122       this.conf = conf;
123       this.fs = fs;
124     }
125 
126     /** @return the snapshot descriptor */
127     public SnapshotDescription getSnapshotDescription() {
128       return this.snapshot;
129     }
130 
131     /** @return true if the snapshot is corrupted */
132     public boolean isSnapshotCorrupted() {
133       return hfilesMissing.get() > 0 || logsMissing.get() > 0;
134     }
135 
136     /** @return the number of available store files */
137     public int getStoreFilesCount() {
138       return hfilesCount.get() + hfileArchiveCount.get();
139     }
140 
141     /** @return the number of available store files in the archive */
142     public int getArchivedStoreFilesCount() {
143       return hfileArchiveCount.get();
144     }
145 
146     /** @return the number of available log files */
147     public int getLogsCount() {
148       return logsCount.get();
149     }
150 
151     /** @return the number of missing store files */
152     public int getMissingStoreFilesCount() {
153       return hfilesMissing.get();
154     }
155 
156     /** @return the number of missing log files */
157     public int getMissingLogsCount() {
158       return logsMissing.get();
159     }
160 
161     /** @return the total size of the store files referenced by the snapshot */
162     public long getStoreFilesSize() {
163       return hfileSize.get() + hfileArchiveSize.get();
164     }
165 
166     /** @return the total size of the store files shared */
167     public long getSharedStoreFilesSize() {
168       return hfileSize.get();
169     }
170 
171     /** @return the total size of the store files in the archive */
172     public long getArchivedStoreFileSize() {
173       return hfileArchiveSize.get();
174     }
175 
176     /** @return the percentage of the shared store files */
177     public float getSharedStoreFilePercentage() {
178       return ((float)hfileSize.get() / (hfileSize.get() + hfileArchiveSize.get())) * 100;
179     }
180 
181     /** @return the total log size */
182     public long getLogsSize() {
183       return logSize.get();
184     }
185 
186     /**
187      * Add the specified store file to the stats
188      * @param region region encoded Name
189      * @param family family name
190      * @param hfile store file name
191      * @return the store file information
192      */
193     FileInfo addStoreFile(final HRegionInfo region, final String family,
194         final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
195       HFileLink link = HFileLink.create(conf, snapshotTable, region.getEncodedName(),
196                                         family, storeFile.getName());
197       boolean inArchive = false;
198       long size = -1;
199       try {
200         if ((inArchive = fs.exists(link.getArchivePath()))) {
201           size = fs.getFileStatus(link.getArchivePath()).getLen();
202           hfileArchiveSize.addAndGet(size);
203           hfileArchiveCount.incrementAndGet();
204         } else {
205           size = link.getFileStatus(fs).getLen();
206           hfileSize.addAndGet(size);
207           hfilesCount.incrementAndGet();
208         }
209       } catch (FileNotFoundException e) {
210         hfilesMissing.incrementAndGet();
211       }
212       return new FileInfo(inArchive, size);
213     }
214 
215     /**
216      * Add the specified log file to the stats
217      * @param server server name
218      * @param logfile log file name
219      * @return the log information
220      */
221     FileInfo addLogFile(final String server, final String logfile) throws IOException {
222       HLogLink logLink = new HLogLink(conf, server, logfile);
223       long size = -1;
224       try {
225         size = logLink.getFileStatus(fs).getLen();
226         logSize.addAndGet(size);
227         logsCount.incrementAndGet();
228       } catch (FileNotFoundException e) {
229         logsMissing.incrementAndGet();
230       }
231       return new FileInfo(false, size);
232     }
233   }
234 
235   private FileSystem fs;
236   private Path rootDir;
237 
238   private SnapshotManifest snapshotManifest;
239 
240   @Override
241   public int run(String[] args) throws IOException, InterruptedException {
242     final Configuration conf = getConf();
243     boolean listSnapshots = false;
244     String snapshotName = null;
245     boolean showSchema = false;
246     boolean showFiles = false;
247     boolean showStats = false;
248 
249     // Process command line args
250     for (int i = 0; i < args.length; i++) {
251       String cmd = args[i];
252       try {
253         if (cmd.equals("-snapshot")) {
254           snapshotName = args[++i];
255         } else if (cmd.equals("-files")) {
256           showFiles = true;
257           showStats = true;
258         } else if (cmd.equals("-stats")) {
259           showStats = true;
260         } else if (cmd.equals("-schema")) {
261           showSchema = true;
262         } else if (cmd.equals("-remote-dir")) {
263           Path sourceDir = new Path(args[++i]);
264           URI defaultFs = sourceDir.getFileSystem(conf).getUri();
265           FSUtils.setFsDefault(conf, new Path(defaultFs));
266           FSUtils.setRootDir(conf, sourceDir);
267         } else if (cmd.equals("-list-snapshots")) {
268           listSnapshots = true;
269         } else if (cmd.equals("-h") || cmd.equals("--help")) {
270           printUsageAndExit();
271         } else {
272           System.err.println("UNEXPECTED: " + cmd);
273           printUsageAndExit();
274         }
275       } catch (Exception e) {
276         printUsageAndExit();
277       }
278     }
279 
280     // List Available Snapshots
281     if (listSnapshots) {
282       SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
283       System.out.printf("%-20s | %-20s | %s%n", "SNAPSHOT", "CREATION TIME", "TABLE NAME");
284       for (SnapshotDescription desc: getSnapshotList(conf)) {
285         System.out.printf("%-20s | %20s | %s%n",
286                           desc.getName(),
287                           df.format(new Date(desc.getCreationTime())),
288                           desc.getTable());
289       }
290       return 0;
291     }
292 
293     if (snapshotName == null) {
294       System.err.println("Missing snapshot name!");
295       printUsageAndExit();
296       return 1;
297     }
298 
299     rootDir = FSUtils.getRootDir(conf);
300     fs = FileSystem.get(rootDir.toUri(), conf);
301     LOG.debug("fs=" + fs.getUri().toString() + " root=" + rootDir);
302 
303     // Load snapshot information
304     if (!loadSnapshotInfo(snapshotName)) {
305       System.err.println("Snapshot '" + snapshotName + "' not found!");
306       return 1;
307     }
308 
309     printInfo();
310     if (showSchema) printSchema();
311     printFiles(showFiles, showStats);
312 
313     return 0;
314   }
315 
316   /**
317    * Load snapshot info and table descriptor for the specified snapshot
318    * @param snapshotName name of the snapshot to load
319    * @return false if snapshot is not found
320    */
321   private boolean loadSnapshotInfo(final String snapshotName) throws IOException {
322     Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
323     if (!fs.exists(snapshotDir)) {
324       LOG.warn("Snapshot '" + snapshotName + "' not found in: " + snapshotDir);
325       return false;
326     }
327 
328     SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
329     snapshotManifest = SnapshotManifest.open(getConf(), fs, snapshotDir, snapshotDesc);
330     return true;
331   }
332 
333   /**
334    * Dump the {@link SnapshotDescription}
335    */
336   private void printInfo() {
337     SnapshotDescription snapshotDesc = snapshotManifest.getSnapshotDescription();
338     SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
339     System.out.println("Snapshot Info");
340     System.out.println("----------------------------------------");
341     System.out.println("   Name: " + snapshotDesc.getName());
342     System.out.println("   Type: " + snapshotDesc.getType());
343     System.out.println("  Table: " + snapshotDesc.getTable());
344     System.out.println(" Format: " + snapshotDesc.getVersion());
345     System.out.println("Created: " + df.format(new Date(snapshotDesc.getCreationTime())));
346     System.out.println();
347   }
348 
349   /**
350    * Dump the {@link HTableDescriptor}
351    */
352   private void printSchema() {
353     System.out.println("Table Descriptor");
354     System.out.println("----------------------------------------");
355     System.out.println(snapshotManifest.getTableDescriptor().toString());
356     System.out.println();
357   }
358 
359   /**
360    * Collect the hfiles and logs statistics of the snapshot and
361    * dump the file list if requested and the collected information.
362    */
363   private void printFiles(final boolean showFiles, final boolean showStats) throws IOException {
364     if (showFiles) {
365       System.out.println("Snapshot Files");
366       System.out.println("----------------------------------------");
367     }
368 
369     // Collect information about hfiles and logs in the snapshot
370     final SnapshotDescription snapshotDesc = snapshotManifest.getSnapshotDescription();
371     final String table = snapshotDesc.getTable();
372     final SnapshotStats stats = new SnapshotStats(this.getConf(), this.fs, snapshotDesc);
373     SnapshotReferenceUtil.concurrentVisitReferencedFiles(getConf(), fs, snapshotManifest,
374       new SnapshotReferenceUtil.SnapshotVisitor() {
375         @Override
376         public void storeFile(final HRegionInfo regionInfo, final String family,
377             final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
378           if (storeFile.hasReference()) return;
379 
380           SnapshotStats.FileInfo info = stats.addStoreFile(regionInfo, family, storeFile);
381           if (showFiles) {
382             System.out.printf("%8s %s/%s/%s/%s %s%n",
383               (info.isMissing() ? "-" : StringUtils.humanReadableInt(info.getSize())),
384               table, regionInfo.getEncodedName(), family, storeFile.getName(),
385               (info.inArchive() ? "(archive)" : info.isMissing() ? "(NOT FOUND)" : ""));
386           }
387         }
388 
389         @Override
390         public void logFile (final String server, final String logfile)
391             throws IOException {
392           SnapshotStats.FileInfo info = stats.addLogFile(server, logfile);
393 
394           if (showFiles) {
395             System.out.printf("%8s log %s on server %s %s%n",
396               (info.isMissing() ? "-" : StringUtils.humanReadableInt(info.getSize())),
397               logfile, server,
398               (info.isMissing() ? "(NOT FOUND)" : ""));
399           }
400         }
401     });
402 
403     // Dump the stats
404     System.out.println();
405     if (stats.isSnapshotCorrupted()) {
406       System.out.println("**************************************************************");
407       System.out.printf("BAD SNAPSHOT: %d hfile(s) and %d log(s) missing.%n",
408         stats.getMissingStoreFilesCount(), stats.getMissingLogsCount());
409       System.out.println("**************************************************************");
410     }
411 
412     if (showStats) {
413       System.out.printf("%d HFiles (%d in archive), total size %s (%.2f%% %s shared with the source table)%n",
414         stats.getStoreFilesCount(), stats.getArchivedStoreFilesCount(),
415         StringUtils.humanReadableInt(stats.getStoreFilesSize()),
416         stats.getSharedStoreFilePercentage(),
417         StringUtils.humanReadableInt(stats.getSharedStoreFilesSize())
418       );
419       System.out.printf("%d Logs, total size %s%n",
420         stats.getLogsCount(), StringUtils.humanReadableInt(stats.getLogsSize()));
421       System.out.println();
422     }
423   }
424 
425   private void printUsageAndExit() {
426     System.err.printf("Usage: bin/hbase %s [options]%n", getClass().getName());
427     System.err.println(" where [options] are:");
428     System.err.println("  -h|-help                Show this help and exit.");
429     System.err.println("  -remote-dir             Root directory that contains the snapshots.");
430     System.err.println("  -list-snapshots         List all the available snapshots and exit.");
431     System.err.println("  -snapshot NAME          Snapshot to examine.");
432     System.err.println("  -files                  Files and logs list.");
433     System.err.println("  -stats                  Files and logs stats.");
434     System.err.println("  -schema                 Describe the snapshotted table.");
435     System.err.println();
436     System.err.println("Examples:");
437     System.err.println("  hbase " + getClass() + " \\");
438     System.err.println("    -snapshot MySnapshot -files");
439     System.exit(1);
440   }
441 
442   /**
443    * Returns the snapshot stats
444    * @param conf the {@link Configuration} to use
445    * @param snapshot {@link SnapshotDescription} to get stats from
446    * @return the snapshot stats
447    */
448   public static SnapshotStats getSnapshotStats(final Configuration conf,
449       final SnapshotDescription snapshot) throws IOException {
450     Path rootDir = FSUtils.getRootDir(conf);
451     FileSystem fs = FileSystem.get(rootDir.toUri(), conf);
452     Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
453     SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshot);
454     final SnapshotStats stats = new SnapshotStats(conf, fs, snapshot);
455     SnapshotReferenceUtil.concurrentVisitReferencedFiles(conf, fs, manifest,
456       new SnapshotReferenceUtil.SnapshotVisitor() {
457         @Override
458         public void storeFile(final HRegionInfo regionInfo, final String family,
459             final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
460           if (!storeFile.hasReference()) {
461             stats.addStoreFile(regionInfo, family, storeFile);
462           }
463         }
464 
465         @Override
466         public void logFile (final String server, final String logfile) throws IOException {
467           stats.addLogFile(server, logfile);
468         }
469     });
470     return stats;
471   }
472 
473   /**
474    * Returns the list of available snapshots in the specified location
475    * @param conf the {@link Configuration} to use
476    * @return the list of snapshots
477    */
478   public static List<SnapshotDescription> getSnapshotList(final Configuration conf)
479       throws IOException {
480     Path rootDir = FSUtils.getRootDir(conf);
481     FileSystem fs = FileSystem.get(rootDir.toUri(), conf);
482     Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
483     FileStatus[] snapshots = fs.listStatus(snapshotDir,
484       new SnapshotDescriptionUtils.CompletedSnaphotDirectoriesFilter(fs));
485     List<SnapshotDescription> snapshotLists =
486       new ArrayList<SnapshotDescription>(snapshots.length);
487     for (FileStatus snapshotDirStat: snapshots) {
488       snapshotLists.add(SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDirStat.getPath()));
489     }
490     return snapshotLists;
491   }
492 
493   /**
494    * The guts of the {@link #main} method.
495    * Call this method to avoid the {@link #main(String[])} System.exit.
496    * @param args
497    * @return errCode
498    * @throws Exception
499    */
500   static int innerMain(final String [] args) throws Exception {
501     return ToolRunner.run(HBaseConfiguration.create(), new SnapshotInfo(), args);
502   }
503 
504   public static void main(String[] args) throws Exception {
505      System.exit(innerMain(args));
506   }
507 }