View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.snapshot;
20  
21  import java.io.IOException;
22  import java.io.FileNotFoundException;
23  import java.net.URI;
24  import java.text.SimpleDateFormat;
25  import java.util.ArrayList;
26  import java.util.Date;
27  import java.util.List;
28  import java.util.concurrent.atomic.AtomicInteger;
29  import java.util.concurrent.atomic.AtomicLong;
30  
31  import org.apache.commons.logging.Log;
32  import org.apache.commons.logging.LogFactory;
33  
34  import org.apache.hadoop.fs.Path;
35  import org.apache.hadoop.fs.FileStatus;
36  import org.apache.hadoop.fs.FileSystem;
37  import org.apache.hadoop.classification.InterfaceAudience;
38  import org.apache.hadoop.classification.InterfaceStability;
39  import org.apache.hadoop.conf.Configured;
40  import org.apache.hadoop.hbase.HRegionInfo;
41  import org.apache.hadoop.hbase.TableName;
42  import org.apache.hadoop.util.StringUtils;
43  import org.apache.hadoop.util.Tool;
44  import org.apache.hadoop.util.ToolRunner;
45  
46  import org.apache.hadoop.conf.Configuration;
47  import org.apache.hadoop.hbase.HBaseConfiguration;
48  import org.apache.hadoop.hbase.HTableDescriptor;
49  import org.apache.hadoop.hbase.io.HFileLink;
50  import org.apache.hadoop.hbase.io.HLogLink;
51  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
52  import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
53  import org.apache.hadoop.hbase.util.FSUtils;
54  
55  /**
56   * Tool for dumping snapshot information.
57   * <ol>
58   * <li> Table Descriptor
59   * <li> Snapshot creation time, type, format version, ...
60   * <li> List of hfiles and hlogs
61   * <li> Stats about hfiles and logs sizes, percentage of shared with the source table, ...
62   * </ol>
63   */
64  @InterfaceAudience.Public
65  @InterfaceStability.Evolving
66  public final class SnapshotInfo extends Configured implements Tool {
67    private static final Log LOG = LogFactory.getLog(SnapshotInfo.class);
68  
69    /**
70     * Statistics about the snapshot
71     * <ol>
72     * <li> How many store files and logs are in the archive
73     * <li> How many store files and logs are shared with the table
74     * <li> Total store files and logs size and shared amount
75     * </ol>
76     */
77    public static class SnapshotStats {
78      /** Information about the file referenced by the snapshot */
79      static class FileInfo {
80        private final boolean corrupted;
81        private final boolean inArchive;
82        private final long size;
83  
84        FileInfo(final boolean inArchive, final long size, final boolean corrupted) {
85          this.corrupted = corrupted;
86          this.inArchive = inArchive;
87          this.size = size;
88        }
89  
90        /** @return true if the file is in the archive */
91        public boolean inArchive() {
92          return this.inArchive;
93        }
94  
95        /** @return true if the file is corrupted */
96        public boolean isCorrupted() {
97          return this.corrupted;
98        }
99  
100       /** @return true if the file is missing */
101       public boolean isMissing() {
102         return this.size < 0;
103       }
104 
105       /** @return the file size */
106       public long getSize() {
107         return this.size;
108       }
109 
110       String getStateToString() {
111         if (isCorrupted()) return "CORRUPTED";
112         if (isMissing()) return "NOT FOUND";
113         if (inArchive()) return "archive";
114         return null;
115       }
116     }
117 
118     private AtomicInteger hfileArchiveCount = new AtomicInteger();
119     private AtomicInteger hfilesCorrupted = new AtomicInteger();
120     private AtomicInteger hfilesMissing = new AtomicInteger();
121     private AtomicInteger hfilesCount = new AtomicInteger();
122     private AtomicInteger logsMissing = new AtomicInteger();
123     private AtomicInteger logsCount = new AtomicInteger();
124     private AtomicLong hfileArchiveSize = new AtomicLong();
125     private AtomicLong hfileSize = new AtomicLong();
126     private AtomicLong logSize = new AtomicLong();
127 
128     private final SnapshotDescription snapshot;
129     private final TableName snapshotTable;
130     private final Configuration conf;
131     private final FileSystem fs;
132 
133     SnapshotStats(final Configuration conf, final FileSystem fs, final SnapshotDescription snapshot)
134     {
135       this.snapshot = snapshot;
136       this.snapshotTable = TableName.valueOf(snapshot.getTable());
137       this.conf = conf;
138       this.fs = fs;
139     }
140 
141     /** @return the snapshot descriptor */
142     public SnapshotDescription getSnapshotDescription() {
143       return this.snapshot;
144     }
145 
146     /** @return true if the snapshot is corrupted */
147     public boolean isSnapshotCorrupted() {
148       return hfilesMissing.get() > 0 ||
149              logsMissing.get() > 0 ||
150              hfilesCorrupted.get() > 0;
151     }
152 
153     /** @return the number of available store files */
154     public int getStoreFilesCount() {
155       return hfilesCount.get() + hfileArchiveCount.get();
156     }
157 
158     /** @return the number of available store files in the archive */
159     public int getArchivedStoreFilesCount() {
160       return hfileArchiveCount.get();
161     }
162 
163     /** @return the number of available log files */
164     public int getLogsCount() {
165       return logsCount.get();
166     }
167 
168     /** @return the number of missing store files */
169     public int getMissingStoreFilesCount() {
170       return hfilesMissing.get();
171     }
172 
173     /** @return the number of corrupted store files */
174     public int getCorruptedStoreFilesCount() {
175       return hfilesCorrupted.get();
176     }
177 
178     /** @return the number of missing log files */
179     public int getMissingLogsCount() {
180       return logsMissing.get();
181     }
182 
183     /** @return the total size of the store files referenced by the snapshot */
184     public long getStoreFilesSize() {
185       return hfileSize.get() + hfileArchiveSize.get();
186     }
187 
188     /** @return the total size of the store files shared */
189     public long getSharedStoreFilesSize() {
190       return hfileSize.get();
191     }
192 
193     /** @return the total size of the store files in the archive */
194     public long getArchivedStoreFileSize() {
195       return hfileArchiveSize.get();
196     }
197 
198     /** @return the percentage of the shared store files */
199     public float getSharedStoreFilePercentage() {
200       return ((float)hfileSize.get() / (hfileSize.get() + hfileArchiveSize.get())) * 100;
201     }
202 
203     /** @return the total log size */
204     public long getLogsSize() {
205       return logSize.get();
206     }
207 
208     /**
209      * Add the specified store file to the stats
210      * @param region region encoded Name
211      * @param family family name
212      * @param hfile store file name
213      * @return the store file information
214      */
215     FileInfo addStoreFile(final HRegionInfo region, final String family,
216         final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
217       HFileLink link = HFileLink.create(conf, snapshotTable, region.getEncodedName(),
218                                         family, storeFile.getName());
219       boolean isCorrupted = false;
220       boolean inArchive = false;
221       long size = -1;
222       try {
223         if ((inArchive = fs.exists(link.getArchivePath()))) {
224           size = fs.getFileStatus(link.getArchivePath()).getLen();
225           hfileArchiveSize.addAndGet(size);
226           hfileArchiveCount.incrementAndGet();
227         } else {
228           size = link.getFileStatus(fs).getLen();
229           hfileSize.addAndGet(size);
230           hfilesCount.incrementAndGet();
231         }
232         isCorrupted = (storeFile.hasFileSize() && storeFile.getFileSize() != size);
233         if (isCorrupted) hfilesCorrupted.incrementAndGet();
234       } catch (FileNotFoundException e) {
235         hfilesMissing.incrementAndGet();
236       }
237       return new FileInfo(inArchive, size, isCorrupted);
238     }
239 
240     /**
241      * Add the specified log file to the stats
242      * @param server server name
243      * @param logfile log file name
244      * @return the log information
245      */
246     FileInfo addLogFile(final String server, final String logfile) throws IOException {
247       HLogLink logLink = new HLogLink(conf, server, logfile);
248       long size = -1;
249       try {
250         size = logLink.getFileStatus(fs).getLen();
251         logSize.addAndGet(size);
252         logsCount.incrementAndGet();
253       } catch (FileNotFoundException e) {
254         logsMissing.incrementAndGet();
255       }
256       return new FileInfo(false, size, false);
257     }
258   }
259 
260   private boolean printSizeInBytes = false;
261   private FileSystem fs;
262   private Path rootDir;
263 
264   private SnapshotManifest snapshotManifest;
265 
266   @Override
267   public int run(String[] args) throws IOException, InterruptedException {
268     final Configuration conf = getConf();
269     boolean listSnapshots = false;
270     String snapshotName = null;
271     boolean showSchema = false;
272     boolean showFiles = false;
273     boolean showStats = false;
274 
275     // Process command line args
276     for (int i = 0; i < args.length; i++) {
277       String cmd = args[i];
278       try {
279         if (cmd.equals("-snapshot")) {
280           snapshotName = args[++i];
281         } else if (cmd.equals("-files")) {
282           showFiles = true;
283           showStats = true;
284         } else if (cmd.equals("-stats")) {
285           showStats = true;
286         } else if (cmd.equals("-schema")) {
287           showSchema = true;
288         } else if (cmd.equals("-remote-dir")) {
289           Path sourceDir = new Path(args[++i]);
290           URI defaultFs = sourceDir.getFileSystem(conf).getUri();
291           FSUtils.setFsDefault(conf, new Path(defaultFs));
292           FSUtils.setRootDir(conf, sourceDir);
293         } else if (cmd.equals("-list-snapshots")) {
294           listSnapshots = true;
295         } else if (cmd.equals("-size-in-bytes")) {
296           printSizeInBytes = true;
297         } else if (cmd.equals("-h") || cmd.equals("--help")) {
298           printUsageAndExit();
299         } else {
300           System.err.println("UNEXPECTED: " + cmd);
301           printUsageAndExit();
302         }
303       } catch (Exception e) {
304         printUsageAndExit();
305       }
306     }
307 
308     // List Available Snapshots
309     if (listSnapshots) {
310       SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
311       System.out.printf("%-20s | %-20s | %s%n", "SNAPSHOT", "CREATION TIME", "TABLE NAME");
312       for (SnapshotDescription desc: getSnapshotList(conf)) {
313         System.out.printf("%-20s | %20s | %s%n",
314                           desc.getName(),
315                           df.format(new Date(desc.getCreationTime())),
316                           desc.getTable());
317       }
318       return 0;
319     }
320 
321     if (snapshotName == null) {
322       System.err.println("Missing snapshot name!");
323       printUsageAndExit();
324       return 1;
325     }
326 
327     rootDir = FSUtils.getRootDir(conf);
328     fs = FileSystem.get(rootDir.toUri(), conf);
329     LOG.debug("fs=" + fs.getUri().toString() + " root=" + rootDir);
330 
331     // Load snapshot information
332     if (!loadSnapshotInfo(snapshotName)) {
333       System.err.println("Snapshot '" + snapshotName + "' not found!");
334       return 1;
335     }
336 
337     printInfo();
338     if (showSchema) printSchema();
339     printFiles(showFiles, showStats);
340 
341     return 0;
342   }
343 
344   /**
345    * Load snapshot info and table descriptor for the specified snapshot
346    * @param snapshotName name of the snapshot to load
347    * @return false if snapshot is not found
348    */
349   private boolean loadSnapshotInfo(final String snapshotName) throws IOException {
350     Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
351     if (!fs.exists(snapshotDir)) {
352       LOG.warn("Snapshot '" + snapshotName + "' not found in: " + snapshotDir);
353       return false;
354     }
355 
356     SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
357     snapshotManifest = SnapshotManifest.open(getConf(), fs, snapshotDir, snapshotDesc);
358     return true;
359   }
360 
361   /**
362    * Dump the {@link SnapshotDescription}
363    */
364   private void printInfo() {
365     SnapshotDescription snapshotDesc = snapshotManifest.getSnapshotDescription();
366     SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
367     System.out.println("Snapshot Info");
368     System.out.println("----------------------------------------");
369     System.out.println("   Name: " + snapshotDesc.getName());
370     System.out.println("   Type: " + snapshotDesc.getType());
371     System.out.println("  Table: " + snapshotDesc.getTable());
372     System.out.println(" Format: " + snapshotDesc.getVersion());
373     System.out.println("Created: " + df.format(new Date(snapshotDesc.getCreationTime())));
374     System.out.println();
375   }
376 
377   /**
378    * Dump the {@link HTableDescriptor}
379    */
380   private void printSchema() {
381     System.out.println("Table Descriptor");
382     System.out.println("----------------------------------------");
383     System.out.println(snapshotManifest.getTableDescriptor().toString());
384     System.out.println();
385   }
386 
387   /**
388    * Collect the hfiles and logs statistics of the snapshot and
389    * dump the file list if requested and the collected information.
390    */
391   private void printFiles(final boolean showFiles, final boolean showStats) throws IOException {
392     if (showFiles) {
393       System.out.println("Snapshot Files");
394       System.out.println("----------------------------------------");
395     }
396 
397     // Collect information about hfiles and logs in the snapshot
398     final SnapshotDescription snapshotDesc = snapshotManifest.getSnapshotDescription();
399     final String table = snapshotDesc.getTable();
400     final SnapshotStats stats = new SnapshotStats(this.getConf(), this.fs, snapshotDesc);
401     SnapshotReferenceUtil.concurrentVisitReferencedFiles(getConf(), fs, snapshotManifest,
402       new SnapshotReferenceUtil.SnapshotVisitor() {
403         @Override
404         public void storeFile(final HRegionInfo regionInfo, final String family,
405             final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
406           if (storeFile.hasReference()) return;
407 
408           SnapshotStats.FileInfo info = stats.addStoreFile(regionInfo, family, storeFile);
409           if (showFiles) {
410             String state = info.getStateToString();
411             System.out.printf("%8s %s/%s/%s/%s %s%n",
412               (info.isMissing() ? "-" : fileSizeToString(info.getSize())),
413               table, regionInfo.getEncodedName(), family, storeFile.getName(),
414               state == null ? "" : "(" + state + ")");
415           }
416         }
417 
418         @Override
419         public void logFile (final String server, final String logfile)
420             throws IOException {
421           SnapshotStats.FileInfo info = stats.addLogFile(server, logfile);
422 
423           if (showFiles) {
424             String state = info.getStateToString();
425             System.out.printf("%8s log %s on server %s (%s)%n",
426               (info.isMissing() ? "-" : fileSizeToString(info.getSize())),
427               logfile, server,
428               state == null ? "" : "(" + state + ")");
429           }
430         }
431     });
432 
433     // Dump the stats
434     System.out.println();
435     if (stats.isSnapshotCorrupted()) {
436       System.out.println("**************************************************************");
437       System.out.printf("BAD SNAPSHOT: %d hfile(s) and %d log(s) missing.%n",
438         stats.getMissingStoreFilesCount(), stats.getMissingLogsCount());
439       System.out.printf("              %d hfile(s) corrupted.%n",
440         stats.getCorruptedStoreFilesCount());
441       System.out.println("**************************************************************");
442     }
443 
444     if (showStats) {
445       System.out.printf("%d HFiles (%d in archive), total size %s (%.2f%% %s shared with the source table)%n",
446         stats.getStoreFilesCount(), stats.getArchivedStoreFilesCount(),
447         fileSizeToString(stats.getStoreFilesSize()),
448         stats.getSharedStoreFilePercentage(),
449         fileSizeToString(stats.getSharedStoreFilesSize())
450       );
451       System.out.printf("%d Logs, total size %s%n",
452         stats.getLogsCount(), fileSizeToString(stats.getLogsSize()));
453       System.out.println();
454     }
455   }
456 
457   private String fileSizeToString(long size) {
458     return printSizeInBytes ? Long.toString(size) : StringUtils.humanReadableInt(size);
459   }
460 
461   private void printUsageAndExit() {
462     System.err.printf("Usage: bin/hbase %s [options]%n", getClass().getName());
463     System.err.println(" where [options] are:");
464     System.err.println("  -h|-help                Show this help and exit.");
465     System.err.println("  -remote-dir             Root directory that contains the snapshots.");
466     System.err.println("  -list-snapshots         List all the available snapshots and exit.");
467     System.err.println("  -size-in-bytes          Print the size of the files in bytes.");
468     System.err.println("  -snapshot NAME          Snapshot to examine.");
469     System.err.println("  -files                  Files and logs list.");
470     System.err.println("  -stats                  Files and logs stats.");
471     System.err.println("  -schema                 Describe the snapshotted table.");
472     System.err.println();
473     System.err.println("Examples:");
474     System.err.println("  hbase " + getClass() + " \\");
475     System.err.println("    -snapshot MySnapshot -files");
476     System.exit(1);
477   }
478 
479   /**
480    * Returns the snapshot stats
481    * @param conf the {@link Configuration} to use
482    * @param snapshot {@link SnapshotDescription} to get stats from
483    * @return the snapshot stats
484    */
485   public static SnapshotStats getSnapshotStats(final Configuration conf,
486       final SnapshotDescription snapshot) throws IOException {
487     Path rootDir = FSUtils.getRootDir(conf);
488     FileSystem fs = FileSystem.get(rootDir.toUri(), conf);
489     Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
490     SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshot);
491     final SnapshotStats stats = new SnapshotStats(conf, fs, snapshot);
492     SnapshotReferenceUtil.concurrentVisitReferencedFiles(conf, fs, manifest,
493       new SnapshotReferenceUtil.SnapshotVisitor() {
494         @Override
495         public void storeFile(final HRegionInfo regionInfo, final String family,
496             final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
497           if (!storeFile.hasReference()) {
498             stats.addStoreFile(regionInfo, family, storeFile);
499           }
500         }
501 
502         @Override
503         public void logFile (final String server, final String logfile) throws IOException {
504           stats.addLogFile(server, logfile);
505         }
506     });
507     return stats;
508   }
509 
510   /**
511    * Returns the list of available snapshots in the specified location
512    * @param conf the {@link Configuration} to use
513    * @return the list of snapshots
514    */
515   public static List<SnapshotDescription> getSnapshotList(final Configuration conf)
516       throws IOException {
517     Path rootDir = FSUtils.getRootDir(conf);
518     FileSystem fs = FileSystem.get(rootDir.toUri(), conf);
519     Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
520     FileStatus[] snapshots = fs.listStatus(snapshotDir,
521       new SnapshotDescriptionUtils.CompletedSnaphotDirectoriesFilter(fs));
522     List<SnapshotDescription> snapshotLists =
523       new ArrayList<SnapshotDescription>(snapshots.length);
524     for (FileStatus snapshotDirStat: snapshots) {
525       snapshotLists.add(SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDirStat.getPath()));
526     }
527     return snapshotLists;
528   }
529 
530   /**
531    * The guts of the {@link #main} method.
532    * Call this method to avoid the {@link #main(String[])} System.exit.
533    * @param args
534    * @return errCode
535    * @throws Exception
536    */
537   static int innerMain(final String [] args) throws Exception {
538     return ToolRunner.run(HBaseConfiguration.create(), new SnapshotInfo(), args);
539   }
540 
541   public static void main(String[] args) throws Exception {
542      System.exit(innerMain(args));
543   }
544 }