View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.snapshot;
20  
21  import java.io.IOException;
22  import java.io.FileNotFoundException;
23  import java.net.URI;
24  import java.text.SimpleDateFormat;
25  import java.util.concurrent.atomic.AtomicInteger;
26  import java.util.concurrent.atomic.AtomicLong;
27  import java.util.ArrayList;
28  import java.util.Date;
29  import java.util.List;
30  
31  import org.apache.commons.logging.Log;
32  import org.apache.commons.logging.LogFactory;
33  
34  import org.apache.hadoop.fs.Path;
35  import org.apache.hadoop.fs.FileStatus;
36  import org.apache.hadoop.fs.FileSystem;
37  import org.apache.hadoop.classification.InterfaceAudience;
38  import org.apache.hadoop.classification.InterfaceStability;
39  import org.apache.hadoop.conf.Configured;
40  import org.apache.hadoop.util.StringUtils;
41  import org.apache.hadoop.util.Tool;
42  import org.apache.hadoop.util.ToolRunner;
43  
44  import org.apache.hadoop.conf.Configuration;
45  import org.apache.hadoop.hbase.HBaseConfiguration;
46  import org.apache.hadoop.hbase.HTableDescriptor;
47  import org.apache.hadoop.hbase.io.HFileLink;
48  import org.apache.hadoop.hbase.io.HLogLink;
49  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
50  import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
51  import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil;
52  import org.apache.hadoop.hbase.util.Bytes;
53  import org.apache.hadoop.hbase.util.FSUtils;
54  import org.apache.hadoop.hbase.util.FSTableDescriptors;
55  
56  /**
57   * Tool for dumping snapshot information.
58   * <ol>
59   * <li> Table Descriptor
60   * <li> Snapshot creation time, type, format version, ...
61   * <li> List of hfiles and hlogs
62   * <li> Stats about hfiles and logs sizes, percentage of shared with the source table, ...
63   * </ol>
64   */
65  @InterfaceAudience.Public
66  @InterfaceStability.Evolving
67  public final class SnapshotInfo extends Configured implements Tool {
68    private static final Log LOG = LogFactory.getLog(SnapshotInfo.class);
69  
70    /**
71     * Statistics about the snapshot
72     * <ol>
73     * <li> How many store files and logs are in the archive
74     * <li> How many store files and logs are shared with the table
75     * <li> Total store files and logs size and shared amount
76     * </ol>
77     */
78    public static class SnapshotStats {
79      /** Information about the file referenced by the snapshot */
80      static class FileInfo {
81        private final boolean inArchive;
82        private final long size;
83  
84        FileInfo(final boolean inArchive, final long size) {
85          this.inArchive = inArchive;
86          this.size = size;
87        }
88  
89        /** @return true if the file is in the archive */
90        public boolean inArchive() {
91          return this.inArchive;
92        }
93  
94        /** @return true if the file is missing */
95        public boolean isMissing() {
96          return this.size < 0;
97        }
98  
99        /** @return the file size */
100       public long getSize() {
101         return this.size;
102       }
103 
104       String getStateToString() {
105         if (isMissing()) return "NOT FOUND";
106         if (inArchive()) return "archive";
107         return null;
108       }
109     }
110 
111     private int hfileArchiveCount = 0;
112     private int hfilesMissing = 0;
113     private int hfilesCount = 0;
114     private int logsMissing = 0;
115     private int logsCount = 0;
116     private long hfileArchiveSize = 0;
117     private long hfileSize = 0;
118     private long logSize = 0;
119 
120     private final SnapshotDescription snapshot;
121     private final Configuration conf;
122     private final FileSystem fs;
123 
124     SnapshotStats(final Configuration conf, final FileSystem fs, final SnapshotDescription snapshot)
125     {
126       this.snapshot = snapshot;
127       this.conf = conf;
128       this.fs = fs;
129     }
130 
131     /** @return the snapshot descriptor */
132     public SnapshotDescription getSnapshotDescription() {
133       return this.snapshot;
134     }
135 
136     /** @return true if the snapshot is corrupted */
137     public boolean isSnapshotCorrupted() {
138       return hfilesMissing > 0 || logsMissing > 0;
139     }
140 
141     /** @return the number of available store files */
142     public int getStoreFilesCount() {
143       return hfilesCount + hfileArchiveCount;
144     }
145 
146     /** @return the number of available store files in the archive */
147     public int getArchivedStoreFilesCount() {
148       return hfileArchiveCount;
149     }
150 
151     /** @return the number of available log files */
152     public int getLogsCount() {
153       return logsCount;
154     }
155 
156     /** @return the number of missing store files */
157     public int getMissingStoreFilesCount() {
158       return hfilesMissing;
159     }
160 
161     /** @return the number of missing log files */
162     public int getMissingLogsCount() {
163       return logsMissing;
164     }
165 
166     /** @return the total size of the store files referenced by the snapshot */
167     public long getStoreFilesSize() {
168       return hfileSize + hfileArchiveSize;
169     }
170 
171     /** @return the total size of the store files shared */
172     public long getSharedStoreFilesSize() {
173       return hfileSize;
174     }
175 
176     /** @return the total size of the store files in the archive */
177     public long getArchivedStoreFileSize() {
178       return hfileArchiveSize;
179     }
180 
181     /** @return the percentage of the shared store files */
182     public float getSharedStoreFilePercentage() {
183       return ((float)hfileSize / (hfileSize + hfileArchiveSize)) * 100;
184     }
185 
186     /** @return the total log size */
187     public long getLogsSize() {
188       return logSize;
189     }
190 
191     /**
192      * Add the specified store file to the stats
193      * @param region region encoded Name
194      * @param family family name
195      * @param hfile store file name
196      * @return the store file information
197      */
198     FileInfo addStoreFile(final String region, final String family, final String hfile)
199           throws IOException {
200       String table = this.snapshot.getTable();
201       HFileLink link = HFileLink.create(conf, table, region, family, hfile);
202       boolean inArchive = false;
203       long size = -1;
204       try {
205         if ((inArchive = fs.exists(link.getArchivePath()))) {
206           size = fs.getFileStatus(link.getArchivePath()).getLen();
207           hfileArchiveSize += size;
208           hfileArchiveCount++;
209         } else {
210           size = link.getFileStatus(fs).getLen();
211           hfileSize += size;
212           hfilesCount++;
213         }
214       } catch (FileNotFoundException e) {
215         hfilesMissing++;
216       }
217       return new FileInfo(inArchive, size);
218     }
219 
220     /**
221      * Add the specified recovered.edits file to the stats
222      * @param region region encoded name
223      * @param logfile log file name
224      * @return the recovered.edits information
225      */
226     FileInfo addRecoveredEdits(final String region, final String logfile) throws IOException {
227       Path rootDir = FSUtils.getRootDir(conf);
228       Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
229       Path path = SnapshotReferenceUtil.getRecoveredEdits(snapshotDir, region, logfile);
230       long size = fs.getFileStatus(path).getLen();
231       logSize += size;
232       logsCount++;
233       return new FileInfo(true, size);
234     }
235 
236     /**
237      * Add the specified log file to the stats
238      * @param server server name
239      * @param logfile log file name
240      * @return the log information
241      */
242     FileInfo addLogFile(final String server, final String logfile) throws IOException {
243       HLogLink logLink = new HLogLink(conf, server, logfile);
244       long size = -1;
245       try {
246         size = logLink.getFileStatus(fs).getLen();
247         logSize += size;
248         logsCount++;
249       } catch (FileNotFoundException e) {
250         logsMissing++;
251       }
252       return new FileInfo(false, size);
253     }
254   }
255 
256   private boolean printSizeInBytes = false;
257   private FileSystem fs;
258   private Path rootDir;
259 
260   private HTableDescriptor snapshotTableDesc;
261   private SnapshotDescription snapshotDesc;
262   private Path snapshotDir;
263 
264   @Override
265   public int run(String[] args) throws IOException, InterruptedException {
266     final Configuration conf = getConf();
267     boolean listSnapshots = false;
268     String snapshotName = null;
269     boolean showSchema = false;
270     boolean showFiles = false;
271     boolean showStats = false;
272 
273     // Process command line args
274     for (int i = 0; i < args.length; i++) {
275       String cmd = args[i];
276       try {
277         if (cmd.equals("-snapshot")) {
278           snapshotName = args[++i];
279         } else if (cmd.equals("-files")) {
280           showFiles = true;
281           showStats = true;
282         } else if (cmd.equals("-stats")) {
283           showStats = true;
284         } else if (cmd.equals("-schema")) {
285           showSchema = true;
286         } else if (cmd.equals("-remote-dir")) {
287           Path sourceDir = new Path(args[++i]);
288           URI defaultFs = sourceDir.getFileSystem(conf).getUri();
289           FSUtils.setFsDefault(conf, new Path(defaultFs));
290           FSUtils.setRootDir(conf, sourceDir);
291         } else if (cmd.equals("-list-snapshots")) {
292           listSnapshots = true;
293         } else if (cmd.equals("-size-in-bytes")) {
294           printSizeInBytes = true;
295         } else if (cmd.equals("-h") || cmd.equals("--help")) {
296           printUsageAndExit();
297         } else {
298           System.err.println("UNEXPECTED: " + cmd);
299           printUsageAndExit();
300         }
301       } catch (Exception e) {
302         printUsageAndExit();
303       }
304     }
305 
306     // List Available Snapshots
307     if (listSnapshots) {
308       SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
309       System.out.printf("%-20s | %-20s | %s%n", "SNAPSHOT", "CREATION TIME", "TABLE NAME");
310       for (SnapshotDescription desc: getSnapshotList(conf)) {
311         System.out.printf("%-20s | %20s | %s%n",
312                           desc.getName(),
313                           df.format(new Date(desc.getCreationTime())),
314                           desc.getTable());
315       }
316       return 0;
317     }
318 
319     if (snapshotName == null) {
320       System.err.println("Missing snapshot name!");
321       printUsageAndExit();
322       return 1;
323     }
324 
325     rootDir = FSUtils.getRootDir(conf);
326     fs = FileSystem.get(rootDir.toUri(), conf);
327     LOG.debug("fs=" + fs.getUri().toString() + " root=" + rootDir);
328 
329     // Load snapshot information
330     if (!loadSnapshotInfo(snapshotName)) {
331       System.err.println("Snapshot '" + snapshotName + "' not found!");
332       return 1;
333     }
334 
335     printInfo();
336     if (showSchema) printSchema();
337     printFiles(showFiles, showStats);
338 
339     return 0;
340   }
341 
342   /**
343    * Load snapshot info and table descriptor for the specified snapshot
344    * @param snapshotName name of the snapshot to load
345    * @return false if snapshot is not found
346    */
347   private boolean loadSnapshotInfo(final String snapshotName) throws IOException {
348     snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
349     if (!fs.exists(snapshotDir)) {
350       LOG.warn("Snapshot '" + snapshotName + "' not found in: " + snapshotDir);
351       return false;
352     }
353 
354     snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
355     snapshotTableDesc = FSTableDescriptors.getTableDescriptor(fs, snapshotDir);
356     return true;
357   }
358 
359   /**
360    * Dump the {@link SnapshotDescription}
361    */
362   private void printInfo() {
363     SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
364     System.out.println("Snapshot Info");
365     System.out.println("----------------------------------------");
366     System.out.println("   Name: " + snapshotDesc.getName());
367     System.out.println("   Type: " + snapshotDesc.getType());
368     System.out.println("  Table: " + snapshotDesc.getTable());
369     System.out.println(" Format: " + snapshotDesc.getVersion());
370     System.out.println("Created: " + df.format(new Date(snapshotDesc.getCreationTime())));
371     System.out.println();
372   }
373 
374   /**
375    * Dump the {@link HTableDescriptor}
376    */
377   private void printSchema() {
378     System.out.println("Table Descriptor");
379     System.out.println("----------------------------------------");
380     System.out.println(snapshotTableDesc.toString());
381     System.out.println();
382   }
383 
384   /**
385    * Collect the hfiles and logs statistics of the snapshot and
386    * dump the file list if requested and the collected information.
387    */
388   private void printFiles(final boolean showFiles, final boolean showStats) throws IOException {
389     if (showFiles) {
390       System.out.println("Snapshot Files");
391       System.out.println("----------------------------------------");
392     }
393 
394     // Collect information about hfiles and logs in the snapshot
395     final String table = this.snapshotDesc.getTable();
396     final SnapshotStats stats = new SnapshotStats(this.getConf(), this.fs, this.snapshotDesc);
397     SnapshotReferenceUtil.visitReferencedFiles(fs, snapshotDir,
398       new SnapshotReferenceUtil.FileVisitor() {
399         public void storeFile (final String region, final String family, final String hfile)
400             throws IOException {
401           SnapshotStats.FileInfo info = stats.addStoreFile(region, family, hfile);
402 
403           if (showFiles) {
404             String state = info.getStateToString();
405             System.out.printf("%8s %s/%s/%s/%s %s%n",
406               (info.isMissing() ? "-" : fileSizeToString(info.getSize())),
407               table, region, family, hfile,
408               state == null ? "" : "(" + state + ")");
409           }
410         }
411 
412         public void recoveredEdits (final String region, final String logfile)
413             throws IOException {
414           SnapshotStats.FileInfo info = stats.addRecoveredEdits(region, logfile);
415 
416           if (showFiles) {
417             System.out.printf("%8s recovered.edits %s on region %s%n",
418               fileSizeToString(info.getSize()), logfile, region);
419           }
420         }
421 
422         public void logFile (final String server, final String logfile)
423             throws IOException {
424           SnapshotStats.FileInfo info = stats.addLogFile(server, logfile);
425 
426           if (showFiles) {
427             String state = info.getStateToString();
428             System.out.printf("%8s log %s on server %s %s%n",
429               (info.isMissing() ? "-" : fileSizeToString(info.getSize())),
430               logfile, server,
431               state == null ? "" : "(" + state + ")");
432           }
433         }
434     });
435 
436     // Dump the stats
437     System.out.println();
438     if (stats.isSnapshotCorrupted()) {
439       System.out.println("**************************************************************");
440       System.out.printf("BAD SNAPSHOT: %d hfile(s) and %d log(s) missing.%n",
441         stats.getMissingStoreFilesCount(), stats.getMissingLogsCount());
442       System.out.println("**************************************************************");
443     }
444 
445     if (showStats) {
446       System.out.printf("%d HFiles (%d in archive), total size %s (%.2f%% %s shared with the source table)%n",
447         stats.getStoreFilesCount(), stats.getArchivedStoreFilesCount(),
448         fileSizeToString(stats.getStoreFilesSize()),
449         stats.getSharedStoreFilePercentage(),
450         fileSizeToString(stats.getSharedStoreFilesSize())
451       );
452       System.out.printf("%d Logs, total size %s%n",
453         stats.getLogsCount(), fileSizeToString(stats.getLogsSize()));
454       System.out.println();
455     }
456   }
457 
458   private String fileSizeToString(long size) {
459     return printSizeInBytes ? Long.toString(size) : StringUtils.humanReadableInt(size);
460   }
461 
462   private void printUsageAndExit() {
463     System.err.printf("Usage: bin/hbase %s [options]%n", getClass().getName());
464     System.err.println(" where [options] are:");
465     System.err.println("  -h|-help                Show this help and exit.");
466     System.err.println("  -remote-dir             Root directory that contains the snapshots.");
467     System.err.println("  -list-snapshots         List all the available snapshots and exit.");
468     System.err.println("  -snapshot NAME          Snapshot to examine.");
469     System.err.println("  -files                  Files and logs list.");
470     System.err.println("  -stats                  Files and logs stats.");
471     System.err.println("  -schema                 Describe the snapshotted table.");
472     System.err.println();
473     System.err.println("Examples:");
474     System.err.println("  hbase " + getClass() + " \\");
475     System.err.println("    -snapshot MySnapshot -files");
476     System.exit(1);
477   }
478 
479   /**
480    * Returns the snapshot stats
481    * @param conf the {@link Configuration} to use
482    * @param snapshot {@link SnapshotDescription} to get stats from
483    * @return the snapshot stats
484    */
485   public static SnapshotStats getSnapshotStats(final Configuration conf,
486       final SnapshotDescription snapshot) throws IOException {
487     Path rootDir = FSUtils.getRootDir(conf);
488     FileSystem fs = FileSystem.get(rootDir.toUri(), conf);
489     Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
490     final SnapshotStats stats = new SnapshotStats(conf, fs, snapshot);
491     SnapshotReferenceUtil.visitReferencedFiles(fs, snapshotDir,
492       new SnapshotReferenceUtil.FileVisitor() {
493         public void storeFile (final String region, final String family, final String hfile)
494             throws IOException {
495           stats.addStoreFile(region, family, hfile);
496         }
497 
498         public void recoveredEdits (final String region, final String logfile) throws IOException {
499           stats.addRecoveredEdits(region, logfile);
500         }
501 
502         public void logFile (final String server, final String logfile) throws IOException {
503           stats.addLogFile(server, logfile);
504         }
505     });
506     return stats;
507   }
508 
509   /**
510    * Returns the list of available snapshots in the specified location
511    * @param conf the {@link Configuration} to use
512    * @return the list of snapshots
513    */
514   public static List<SnapshotDescription> getSnapshotList(final Configuration conf)
515       throws IOException {
516     Path rootDir = FSUtils.getRootDir(conf);
517     FileSystem fs = FileSystem.get(rootDir.toUri(), conf);
518     Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
519     FileStatus[] snapshots = fs.listStatus(snapshotDir,
520       new SnapshotDescriptionUtils.CompletedSnaphotDirectoriesFilter(fs));
521     List<SnapshotDescription> snapshotLists =
522       new ArrayList<SnapshotDescription>(snapshots.length);
523     for (FileStatus snapshotDirStat: snapshots) {
524       snapshotLists.add(SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDirStat.getPath()));
525     }
526     return snapshotLists;
527   }
528 
529   /**
530    * The guts of the {@link #main} method.
531    * Call this method to avoid the {@link #main(String[])} System.exit.
532    * @param args
533    * @return errCode
534    * @throws Exception
535    */
536   static int innerMain(final String [] args) throws Exception {
537     return ToolRunner.run(HBaseConfiguration.create(), new SnapshotInfo(), args);
538   }
539 
540   public static void main(String[] args) throws Exception {
541      System.exit(innerMain(args));
542   }
543 }