View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.snapshot;
20  
21  import java.io.IOException;
22  import java.io.FileNotFoundException;
23  import java.net.URI;
24  import java.text.SimpleDateFormat;
25  import java.util.ArrayList;
26  import java.util.Date;
27  import java.util.List;
28  import java.util.concurrent.atomic.AtomicInteger;
29  import java.util.concurrent.atomic.AtomicLong;
30  
31  import org.apache.commons.logging.Log;
32  import org.apache.commons.logging.LogFactory;
33  
34  import org.apache.hadoop.fs.Path;
35  import org.apache.hadoop.fs.FileStatus;
36  import org.apache.hadoop.fs.FileSystem;
37  import org.apache.hadoop.hbase.classification.InterfaceAudience;
38  import org.apache.hadoop.hbase.classification.InterfaceStability;
39  import org.apache.hadoop.conf.Configured;
40  import org.apache.hadoop.hbase.HRegionInfo;
41  import org.apache.hadoop.hbase.TableName;
42  import org.apache.hadoop.util.StringUtils;
43  import org.apache.hadoop.util.Tool;
44  import org.apache.hadoop.util.ToolRunner;
45  
46  import org.apache.hadoop.conf.Configuration;
47  import org.apache.hadoop.hbase.HBaseConfiguration;
48  import org.apache.hadoop.hbase.io.HFileLink;
49  import org.apache.hadoop.hbase.io.WALLink;
50  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
51  import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
52  import org.apache.hadoop.hbase.util.FSUtils;
53  
54  /**
55   * Tool for dumping snapshot information.
56   * <ol>
57   * <li> Table Descriptor
58   * <li> Snapshot creation time, type, format version, ...
59   * <li> List of hfiles and wals
60   * <li> Stats about hfiles and logs sizes, percentage of shared with the source table, ...
61   * </ol>
62   */
63  @InterfaceAudience.Public
64  @InterfaceStability.Evolving
65  public final class SnapshotInfo extends Configured implements Tool {
66    private static final Log LOG = LogFactory.getLog(SnapshotInfo.class);
67  
68    /**
69     * Statistics about the snapshot
70     * <ol>
71     * <li> How many store files and logs are in the archive
72     * <li> How many store files and logs are shared with the table
73     * <li> Total store files and logs size and shared amount
74     * </ol>
75     */
76    public static class SnapshotStats {
77      /** Information about the file referenced by the snapshot */
78      static class FileInfo {
79        private final boolean corrupted;
80        private final boolean inArchive;
81        private final long size;
82  
83        FileInfo(final boolean inArchive, final long size, final boolean corrupted) {
84          this.corrupted = corrupted;
85          this.inArchive = inArchive;
86          this.size = size;
87        }
88  
89        /** @return true if the file is in the archive */
90        public boolean inArchive() {
91          return this.inArchive;
92        }
93  
94        /** @return true if the file is corrupted */
95        public boolean isCorrupted() {
96          return this.corrupted;
97        }
98  
99        /** @return true if the file is missing */
100       public boolean isMissing() {
101         return this.size < 0;
102       }
103 
104       /** @return the file size */
105       public long getSize() {
106         return this.size;
107       }
108 
109       String getStateToString() {
110         if (isCorrupted()) return "CORRUPTED";
111         if (isMissing()) return "NOT FOUND";
112         if (inArchive()) return "archive";
113         return null;
114       }
115     }
116 
117     private AtomicInteger hfilesArchiveCount = new AtomicInteger();
118     private AtomicInteger hfilesCorrupted = new AtomicInteger();
119     private AtomicInteger hfilesMissing = new AtomicInteger();
120     private AtomicInteger hfilesCount = new AtomicInteger();
121     private AtomicInteger hfilesMobCount = new AtomicInteger();
122     private AtomicInteger logsMissing = new AtomicInteger();
123     private AtomicInteger logsCount = new AtomicInteger();
124     private AtomicLong hfilesArchiveSize = new AtomicLong();
125     private AtomicLong hfilesSize = new AtomicLong();
126     private AtomicLong hfilesMobSize = new AtomicLong();
127     private AtomicLong logSize = new AtomicLong();
128 
129     private final SnapshotDescription snapshot;
130     private final TableName snapshotTable;
131     private final Configuration conf;
132     private final FileSystem fs;
133 
134     SnapshotStats(final Configuration conf, final FileSystem fs, final SnapshotDescription snapshot)
135     {
136       this.snapshot = snapshot;
137       this.snapshotTable = TableName.valueOf(snapshot.getTable());
138       this.conf = conf;
139       this.fs = fs;
140     }
141 
142     /** @return the snapshot descriptor */
143     public SnapshotDescription getSnapshotDescription() {
144       return this.snapshot;
145     }
146 
147     /** @return true if the snapshot is corrupted */
148     public boolean isSnapshotCorrupted() {
149       return hfilesMissing.get() > 0 ||
150              logsMissing.get() > 0 ||
151              hfilesCorrupted.get() > 0;
152     }
153 
154     /** @return the number of available store files */
155     public int getStoreFilesCount() {
156       return hfilesCount.get() + hfilesArchiveCount.get() + hfilesMobCount.get();
157     }
158 
159     /** @return the number of available store files in the archive */
160     public int getArchivedStoreFilesCount() {
161       return hfilesArchiveCount.get();
162     }
163 
164     /** @return the number of available store files in the mob dir */
165     public int getMobStoreFilesCount() { return hfilesMobCount.get(); }
166 
167     /** @return the number of available log files */
168     public int getLogsCount() {
169       return logsCount.get();
170     }
171 
172     /** @return the number of missing store files */
173     public int getMissingStoreFilesCount() {
174       return hfilesMissing.get();
175     }
176 
177     /** @return the number of corrupted store files */
178     public int getCorruptedStoreFilesCount() {
179       return hfilesCorrupted.get();
180     }
181 
182     /** @return the number of missing log files */
183     public int getMissingLogsCount() {
184       return logsMissing.get();
185     }
186 
187     /** @return the total size of the store files referenced by the snapshot */
188     public long getStoreFilesSize() {
189       return hfilesSize.get() + hfilesArchiveSize.get() + hfilesMobSize.get();
190     }
191 
192     /** @return the total size of the store files shared */
193     public long getSharedStoreFilesSize() {
194       return hfilesSize.get();
195     }
196 
197     /** @return the total size of the store files in the archive */
198     public long getArchivedStoreFileSize() {
199       return hfilesArchiveSize.get();
200     }
201 
202     /** @return the total size of the store files in the mob store*/
203     public long getMobStoreFilesSize() { return hfilesMobSize.get(); }
204 
205     /** @return the percentage of the shared store files */
206     public float getSharedStoreFilePercentage() {
207       return ((float) hfilesSize.get() / (getStoreFilesSize())) * 100;
208     }
209 
210     /** @return the percentage of the mob store files */
211     public float getMobStoreFilePercentage() {
212       return ((float) hfilesMobSize.get() / (getStoreFilesSize())) * 100;
213     }
214 
215     /** @return the total log size */
216     public long getLogsSize() {
217       return logSize.get();
218     }
219 
220     /**
221      * Add the specified store file to the stats
222      * @param region region encoded Name
223      * @param family family name
224      * @param storeFile store file name
225      * @return the store file information
226      */
227     FileInfo addStoreFile(final HRegionInfo region, final String family,
228         final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
229       HFileLink link = HFileLink.build(conf, snapshotTable, region.getEncodedName(),
230               family, storeFile.getName());
231       boolean isCorrupted = false;
232       boolean inArchive = false;
233       long size = -1;
234       try {
235         if ((inArchive = fs.exists(link.getArchivePath()))) {
236           size = fs.getFileStatus(link.getArchivePath()).getLen();
237           hfilesArchiveSize.addAndGet(size);
238           hfilesArchiveCount.incrementAndGet();
239         } else if (inArchive = fs.exists(link.getMobPath())) {
240           size = fs.getFileStatus(link.getMobPath()).getLen();
241           hfilesMobSize.addAndGet(size);
242           hfilesMobCount.incrementAndGet();
243         } else {
244           size = link.getFileStatus(fs).getLen();
245           hfilesSize.addAndGet(size);
246           hfilesCount.incrementAndGet();
247         }
248         isCorrupted = (storeFile.hasFileSize() && storeFile.getFileSize() != size);
249         if (isCorrupted) hfilesCorrupted.incrementAndGet();
250       } catch (FileNotFoundException e) {
251         hfilesMissing.incrementAndGet();
252       }
253       return new FileInfo(inArchive, size, isCorrupted);
254     }
255 
256     /**
257      * Add the specified log file to the stats
258      * @param server server name
259      * @param logfile log file name
260      * @return the log information
261      */
262     FileInfo addLogFile(final String server, final String logfile) throws IOException {
263       WALLink logLink = new WALLink(conf, server, logfile);
264       long size = -1;
265       try {
266         size = logLink.getFileStatus(fs).getLen();
267         logSize.addAndGet(size);
268         logsCount.incrementAndGet();
269       } catch (FileNotFoundException e) {
270         logsMissing.incrementAndGet();
271       }
272       return new FileInfo(false, size, false);
273     }
274   }
275 
276   private boolean printSizeInBytes = false;
277   private FileSystem fs;
278   private Path rootDir;
279 
280   private SnapshotManifest snapshotManifest;
281 
282   @Override
283   public int run(String[] args) throws IOException, InterruptedException {
284     final Configuration conf = getConf();
285     boolean listSnapshots = false;
286     String snapshotName = null;
287     boolean showSchema = false;
288     boolean showFiles = false;
289     boolean showStats = false;
290 
291     // Process command line args
292     for (int i = 0; i < args.length; i++) {
293       String cmd = args[i];
294       try {
295         if (cmd.equals("-snapshot")) {
296           snapshotName = args[++i];
297         } else if (cmd.equals("-files")) {
298           showFiles = true;
299           showStats = true;
300         } else if (cmd.equals("-stats")) {
301           showStats = true;
302         } else if (cmd.equals("-schema")) {
303           showSchema = true;
304         } else if (cmd.equals("-remote-dir")) {
305           Path sourceDir = new Path(args[++i]);
306           URI defaultFs = sourceDir.getFileSystem(conf).getUri();
307           FSUtils.setFsDefault(conf, new Path(defaultFs));
308           FSUtils.setRootDir(conf, sourceDir);
309         } else if (cmd.equals("-list-snapshots")) {
310           listSnapshots = true;
311         } else if (cmd.equals("-size-in-bytes")) {
312           printSizeInBytes = true;
313         } else if (cmd.equals("-h") || cmd.equals("--help")) {
314           printUsageAndExit();
315         } else {
316           System.err.println("UNEXPECTED: " + cmd);
317           printUsageAndExit();
318         }
319       } catch (Exception e) {
320         printUsageAndExit();
321       }
322     }
323 
324     // List Available Snapshots
325     if (listSnapshots) {
326       SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
327       System.out.printf("%-20s | %-20s | %s%n", "SNAPSHOT", "CREATION TIME", "TABLE NAME");
328       for (SnapshotDescription desc: getSnapshotList(conf)) {
329         System.out.printf("%-20s | %20s | %s%n",
330                           desc.getName(),
331                           df.format(new Date(desc.getCreationTime())),
332                           desc.getTable());
333       }
334       return 0;
335     }
336 
337     if (snapshotName == null) {
338       System.err.println("Missing snapshot name!");
339       printUsageAndExit();
340       return 1;
341     }
342 
343     rootDir = FSUtils.getRootDir(conf);
344     fs = FileSystem.get(rootDir.toUri(), conf);
345     LOG.debug("fs=" + fs.getUri().toString() + " root=" + rootDir);
346 
347     // Load snapshot information
348     if (!loadSnapshotInfo(snapshotName)) {
349       System.err.println("Snapshot '" + snapshotName + "' not found!");
350       return 1;
351     }
352 
353     printInfo();
354     if (showSchema) printSchema();
355     printFiles(showFiles, showStats);
356 
357     return 0;
358   }
359 
360   /**
361    * Load snapshot info and table descriptor for the specified snapshot
362    * @param snapshotName name of the snapshot to load
363    * @return false if snapshot is not found
364    */
365   private boolean loadSnapshotInfo(final String snapshotName) throws IOException {
366     Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
367     if (!fs.exists(snapshotDir)) {
368       LOG.warn("Snapshot '" + snapshotName + "' not found in: " + snapshotDir);
369       return false;
370     }
371 
372     SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
373     snapshotManifest = SnapshotManifest.open(getConf(), fs, snapshotDir, snapshotDesc);
374     return true;
375   }
376 
377   /**
378    * Dump the {@link SnapshotDescription}
379    */
380   private void printInfo() {
381     SnapshotDescription snapshotDesc = snapshotManifest.getSnapshotDescription();
382     SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
383     System.out.println("Snapshot Info");
384     System.out.println("----------------------------------------");
385     System.out.println("   Name: " + snapshotDesc.getName());
386     System.out.println("   Type: " + snapshotDesc.getType());
387     System.out.println("  Table: " + snapshotDesc.getTable());
388     System.out.println(" Format: " + snapshotDesc.getVersion());
389     System.out.println("Created: " + df.format(new Date(snapshotDesc.getCreationTime())));
390     System.out.println();
391   }
392 
393   /**
394    * Dump the {@link HTableDescriptor}
395    */
396   private void printSchema() {
397     System.out.println("Table Descriptor");
398     System.out.println("----------------------------------------");
399     System.out.println(snapshotManifest.getTableDescriptor().toString());
400     System.out.println();
401   }
402 
403   /**
404    * Collect the hfiles and logs statistics of the snapshot and
405    * dump the file list if requested and the collected information.
406    */
407   private void printFiles(final boolean showFiles, final boolean showStats) throws IOException {
408     if (showFiles) {
409       System.out.println("Snapshot Files");
410       System.out.println("----------------------------------------");
411     }
412 
413     // Collect information about hfiles and logs in the snapshot
414     final SnapshotDescription snapshotDesc = snapshotManifest.getSnapshotDescription();
415     final String table = snapshotDesc.getTable();
416     final SnapshotStats stats = new SnapshotStats(this.getConf(), this.fs, snapshotDesc);
417     SnapshotReferenceUtil.concurrentVisitReferencedFiles(getConf(), fs, snapshotManifest,
418       new SnapshotReferenceUtil.SnapshotVisitor() {
419         @Override
420         public void storeFile(final HRegionInfo regionInfo, final String family,
421             final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
422           if (storeFile.hasReference()) return;
423 
424           SnapshotStats.FileInfo info = stats.addStoreFile(regionInfo, family, storeFile);
425           if (showFiles) {
426             String state = info.getStateToString();
427             System.out.printf("%8s %s/%s/%s/%s %s%n",
428               (info.isMissing() ? "-" : fileSizeToString(info.getSize())),
429               table, regionInfo.getEncodedName(), family, storeFile.getName(),
430               state == null ? "" : "(" + state + ")");
431           }
432         }
433 
434         @Override
435         public void logFile (final String server, final String logfile)
436             throws IOException {
437           SnapshotStats.FileInfo info = stats.addLogFile(server, logfile);
438 
439           if (showFiles) {
440             String state = info.getStateToString();
441             System.out.printf("%8s log %s on server %s (%s)%n",
442               (info.isMissing() ? "-" : fileSizeToString(info.getSize())),
443               logfile, server,
444               state == null ? "" : "(" + state + ")");
445           }
446         }
447     });
448 
449     // Dump the stats
450     System.out.println();
451     if (stats.isSnapshotCorrupted()) {
452       System.out.println("**************************************************************");
453       System.out.printf("BAD SNAPSHOT: %d hfile(s) and %d log(s) missing.%n",
454         stats.getMissingStoreFilesCount(), stats.getMissingLogsCount());
455       System.out.printf("              %d hfile(s) corrupted.%n",
456         stats.getCorruptedStoreFilesCount());
457       System.out.println("**************************************************************");
458     }
459 
460     if (showStats) {
461       System.out.printf("%d HFiles (%d in archive, %d in mob storage), total size %s " +
462               "(%.2f%% %s shared with the source table, %.2f%% %s in mob dir)%n",
463         stats.getStoreFilesCount(), stats.getArchivedStoreFilesCount(),
464         stats.getMobStoreFilesCount(),
465         fileSizeToString(stats.getStoreFilesSize()),
466         stats.getSharedStoreFilePercentage(),
467         fileSizeToString(stats.getSharedStoreFilesSize()),
468         stats.getMobStoreFilePercentage(),
469         fileSizeToString(stats.getMobStoreFilesSize())
470       );
471       System.out.printf("%d Logs, total size %s%n",
472         stats.getLogsCount(), fileSizeToString(stats.getLogsSize()));
473       System.out.println();
474     }
475   }
476 
477   private String fileSizeToString(long size) {
478     return printSizeInBytes ? Long.toString(size) : StringUtils.humanReadableInt(size);
479   }
480 
481   private void printUsageAndExit() {
482     System.err.printf("Usage: bin/hbase %s [options]%n", getClass().getName());
483     System.err.println(" where [options] are:");
484     System.err.println("  -h|-help                Show this help and exit.");
485     System.err.println("  -remote-dir             Root directory that contains the snapshots.");
486     System.err.println("  -list-snapshots         List all the available snapshots and exit.");
487     System.err.println("  -size-in-bytes          Print the size of the files in bytes.");
488     System.err.println("  -snapshot NAME          Snapshot to examine.");
489     System.err.println("  -files                  Files and logs list.");
490     System.err.println("  -stats                  Files and logs stats.");
491     System.err.println("  -schema                 Describe the snapshotted table.");
492     System.err.println();
493     System.err.println("Examples:");
494     System.err.println("  hbase " + getClass() + " \\");
495     System.err.println("    -snapshot MySnapshot -files");
496     System.exit(1);
497   }
498 
499   /**
500    * Returns the snapshot stats
501    * @param conf the {@link Configuration} to use
502    * @param snapshot {@link SnapshotDescription} to get stats from
503    * @return the snapshot stats
504    */
505   public static SnapshotStats getSnapshotStats(final Configuration conf,
506       final SnapshotDescription snapshot) throws IOException {
507     Path rootDir = FSUtils.getRootDir(conf);
508     FileSystem fs = FileSystem.get(rootDir.toUri(), conf);
509     Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
510     SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshot);
511     final SnapshotStats stats = new SnapshotStats(conf, fs, snapshot);
512     SnapshotReferenceUtil.concurrentVisitReferencedFiles(conf, fs, manifest,
513       new SnapshotReferenceUtil.SnapshotVisitor() {
514         @Override
515         public void storeFile(final HRegionInfo regionInfo, final String family,
516             final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
517           if (!storeFile.hasReference()) {
518             stats.addStoreFile(regionInfo, family, storeFile);
519           }
520         }
521 
522         @Override
523         public void logFile (final String server, final String logfile) throws IOException {
524           stats.addLogFile(server, logfile);
525         }
526     });
527     return stats;
528   }
529 
530   /**
531    * Returns the list of available snapshots in the specified location
532    * @param conf the {@link Configuration} to use
533    * @return the list of snapshots
534    */
535   public static List<SnapshotDescription> getSnapshotList(final Configuration conf)
536       throws IOException {
537     Path rootDir = FSUtils.getRootDir(conf);
538     FileSystem fs = FileSystem.get(rootDir.toUri(), conf);
539     Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
540     FileStatus[] snapshots = fs.listStatus(snapshotDir,
541       new SnapshotDescriptionUtils.CompletedSnaphotDirectoriesFilter(fs));
542     List<SnapshotDescription> snapshotLists =
543       new ArrayList<SnapshotDescription>(snapshots.length);
544     for (FileStatus snapshotDirStat: snapshots) {
545       snapshotLists.add(SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDirStat.getPath()));
546     }
547     return snapshotLists;
548   }
549 
550   /**
551    * The guts of the {@link #main} method.
552    * Call this method to avoid the {@link #main(String[])} System.exit.
553    * @param args
554    * @return errCode
555    * @throws Exception
556    */
557   static int innerMain(final String [] args) throws Exception {
558     return ToolRunner.run(HBaseConfiguration.create(), new SnapshotInfo(), args);
559   }
560 
561   public static void main(String[] args) throws Exception {
562      System.exit(innerMain(args));
563   }
564 }