View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.snapshot;
20  
21  import java.io.IOException;
22  import java.io.FileNotFoundException;
23  import java.net.URI;
24  import java.text.SimpleDateFormat;
25  import java.util.ArrayList;
26  import java.util.Collections;
27  import java.util.Date;
28  import java.util.List;
29  import java.util.Map;
30  import java.util.concurrent.ConcurrentHashMap;
31  import java.util.concurrent.ExecutorService;
32  import java.util.concurrent.atomic.AtomicInteger;
33  import java.util.concurrent.atomic.AtomicLong;
34
35  import org.apache.commons.logging.Log;
36  import org.apache.commons.logging.LogFactory;
37
38  import org.apache.hadoop.fs.Path;
39  import org.apache.hadoop.fs.FileStatus;
40  import org.apache.hadoop.fs.FileSystem;
41  import org.apache.hadoop.hbase.classification.InterfaceAudience;
42  import org.apache.hadoop.hbase.classification.InterfaceStability;
43  import org.apache.hadoop.hbase.client.SnapshotDescription;
44  import org.apache.hadoop.conf.Configured;
45  import org.apache.hadoop.hbase.HRegionInfo;
46  import org.apache.hadoop.hbase.TableName;
47  import org.apache.hadoop.util.StringUtils;
48  import org.apache.hadoop.util.Tool;
49  import org.apache.hadoop.util.ToolRunner;
50
51  import org.apache.hadoop.conf.Configuration;
52  import org.apache.hadoop.hbase.HBaseConfiguration;
53  import org.apache.hadoop.hbase.io.HFileLink;
54  import org.apache.hadoop.hbase.io.WALLink;
55  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
56  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
57  import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
58  import org.apache.hadoop.hbase.util.FSUtils;
59
60  /**
61   * Tool for dumping snapshot information.
62   * <ol>
63   * <li> Table Descriptor
64   * <li> Snapshot creation time, type, format version, ...
65   * <li> List of hfiles and wals
66   * <li> Stats about hfiles and logs sizes, percentage of shared with the source table, ...
67   * </ol>
68   */
69  @InterfaceAudience.Public
70  @InterfaceStability.Evolving
71  public final class SnapshotInfo extends Configured implements Tool {
72    private static final Log LOG = LogFactory.getLog(SnapshotInfo.class);
73
74    /**
75     * Statistics about the snapshot
76     * <ol>
77     * <li> How many store files and logs are in the archive
78     * <li> How many store files and logs are shared with the table
79     * <li> Total store files and logs size and shared amount
80     * </ol>
81     */
82    public static class SnapshotStats {
83      /** Information about the file referenced by the snapshot */
84      static class FileInfo {
85        private final boolean corrupted;
86        private final boolean inArchive;
87        private final long size;
88
89        FileInfo(final boolean inArchive, final long size, final boolean corrupted) {
90          this.corrupted = corrupted;
91          this.inArchive = inArchive;
92          this.size = size;
93        }
94
95        /** @return true if the file is in the archive */
96        public boolean inArchive() {
97          return this.inArchive;
98        }
99
100       /** @return true if the file is corrupted */
101       public boolean isCorrupted() {
102         return this.corrupted;
103       }
104
105       /** @return true if the file is missing */
106       public boolean isMissing() {
107         return this.size < 0;
108       }
109
110       /** @return the file size */
111       public long getSize() {
112         return this.size;
113       }
114
115       String getStateToString() {
116         if (isCorrupted()) return "CORRUPTED";
117         if (isMissing()) return "NOT FOUND";
118         if (inArchive()) return "archive";
119         return null;
120       }
121     }
122
123     private AtomicInteger hfilesArchiveCount = new AtomicInteger();
124     private AtomicInteger hfilesCorrupted = new AtomicInteger();
125     private AtomicInteger hfilesMissing = new AtomicInteger();
126     private AtomicInteger hfilesCount = new AtomicInteger();
127     private AtomicInteger hfilesMobCount = new AtomicInteger();
128     private AtomicInteger logsMissing = new AtomicInteger();
129     private AtomicInteger logsCount = new AtomicInteger();
130     private AtomicLong hfilesArchiveSize = new AtomicLong();
131     private AtomicLong hfilesSize = new AtomicLong();
132     private AtomicLong hfilesMobSize = new AtomicLong();
133     private AtomicLong nonSharedHfilesArchiveSize = new AtomicLong();
134     private AtomicLong logSize = new AtomicLong();
135
136     private final HBaseProtos.SnapshotDescription snapshot;
137     private final TableName snapshotTable;
138     private final Configuration conf;
139     private final FileSystem fs;
140
141     SnapshotStats(final Configuration conf, final FileSystem fs,
142         final SnapshotDescription snapshot)
143     {
144       this.snapshot = ProtobufUtil.createHBaseProtosSnapshotDesc(snapshot);
145       this.snapshotTable = TableName.valueOf(snapshot.getTable());
146       this.conf = conf;
147       this.fs = fs;
148     }
149
150     SnapshotStats(final Configuration conf, final FileSystem fs,
151         final HBaseProtos.SnapshotDescription snapshot) {
152       this.snapshot = snapshot;
153       this.snapshotTable = TableName.valueOf(snapshot.getTable());
154       this.conf = conf;
155       this.fs = fs;
156     }
157
158 
159     /** @return the snapshot descriptor */
160     public SnapshotDescription getSnapshotDescription() {
161       return new SnapshotDescription(this.snapshot.getName(), this.snapshot.getTable(),
162           ProtobufUtil.createSnapshotType(this.snapshot.getType()), this.snapshot.getOwner(),
163           this.snapshot.getCreationTime(), this.snapshot.getVersion());
164     }
165
166     /** @return true if the snapshot is corrupted */
167     public boolean isSnapshotCorrupted() {
168       return hfilesMissing.get() > 0 ||
169              logsMissing.get() > 0 ||
170              hfilesCorrupted.get() > 0;
171     }
172
173     /** @return the number of available store files */
174     public int getStoreFilesCount() {
175       return hfilesCount.get() + hfilesArchiveCount.get() + hfilesMobCount.get();
176     }
177
178     /** @return the number of available store files in the archive */
179     public int getArchivedStoreFilesCount() {
180       return hfilesArchiveCount.get();
181     }
182
183     /** @return the number of available store files in the mob dir */
184     public int getMobStoreFilesCount() { return hfilesMobCount.get(); }
185
186     /** @return the number of available log files */
187     public int getLogsCount() {
188       return logsCount.get();
189     }
190
191     /** @return the number of missing store files */
192     public int getMissingStoreFilesCount() {
193       return hfilesMissing.get();
194     }
195
196     /** @return the number of corrupted store files */
197     public int getCorruptedStoreFilesCount() {
198       return hfilesCorrupted.get();
199     }
200
201     /** @return the number of missing log files */
202     public int getMissingLogsCount() {
203       return logsMissing.get();
204     }
205
206     /** @return the total size of the store files referenced by the snapshot */
207     public long getStoreFilesSize() {
208       return hfilesSize.get() + hfilesArchiveSize.get() + hfilesMobSize.get();
209     }
210
211     /** @return the total size of the store files shared */
212     public long getSharedStoreFilesSize() {
213       return hfilesSize.get();
214     }
215
216     /** @return the total size of the store files in the archive */
217     public long getArchivedStoreFileSize() {
218       return hfilesArchiveSize.get();
219     }
220
221     /** @return the total size of the store files in the mob store*/
222     public long getMobStoreFilesSize() { return hfilesMobSize.get(); }
223
224     /** @return the total size of the store files in the archive which is not shared
225      *    with other snapshots and tables
226      *
227      *    This is only calculated when
228      *  {@link #getSnapshotStats(Configuration, HBaseProtos.SnapshotDescription, Map)}
229      *    is called with a non-null Map
230      */
231     public long getNonSharedArchivedStoreFilesSize() {
232       return nonSharedHfilesArchiveSize.get();
233     }
234
235     /** @return the percentage of the shared store files */
236     public float getSharedStoreFilePercentage() {
237       return ((float) hfilesSize.get() / (getStoreFilesSize())) * 100;
238     }
239
240     /** @return the percentage of the mob store files */
241     public float getMobStoreFilePercentage() {
242       return ((float) hfilesMobSize.get() / (getStoreFilesSize())) * 100;
243     }
244
245     /** @return the total log size */
246     public long getLogsSize() {
247       return logSize.get();
248     }
249
250     /** Check if for a give file in archive, if there are other snapshots/tables still
251      * reference it.
252      * @param filePath file path in archive
253      * @param snapshotFilesMap a map for store files in snapshots about how many snapshots refer
254      *                         to it.
255      * @return true or false
256      */
257     private boolean isArchivedFileStillReferenced(final Path filePath,
258         final Map<Path, Integer> snapshotFilesMap) {
259
260       Integer c = snapshotFilesMap.get(filePath);
261
262       // Check if there are other snapshots or table from clone_snapshot() (via back-reference)
263       // still reference to it.
264       if ((c != null) && (c == 1)) {
265         Path parentDir = filePath.getParent();
266         Path backRefDir = HFileLink.getBackReferencesDir(parentDir, filePath.getName());
267         try {
268           if (FSUtils.listStatus(fs, backRefDir) == null) {
269             return false;
270           }
271         } catch (IOException e) {
272           // For the purpose of this function, IOException is ignored and treated as
273           // the file is still being referenced.
274         }
275       }
276       return true;
277     }
278
279     /**
280      * Add the specified store file to the stats
281      * @param region region encoded Name
282      * @param family family name
283      * @param storeFile store file name
284      * @param filesMap store files map for all snapshots, it may be null
285      * @return the store file information
286      */
287     FileInfo addStoreFile(final HRegionInfo region, final String family,
288         final SnapshotRegionManifest.StoreFile storeFile,
289         final Map<Path, Integer> filesMap) throws IOException {
290       HFileLink link = HFileLink.build(conf, snapshotTable, region.getEncodedName(),
291               family, storeFile.getName());
292       boolean isCorrupted = false;
293       boolean inArchive = false;
294       long size = -1;
295       try {
296         if ((inArchive = fs.exists(link.getArchivePath()))) {
297           size = fs.getFileStatus(link.getArchivePath()).getLen();
298           hfilesArchiveSize.addAndGet(size);
299           hfilesArchiveCount.incrementAndGet();
300
301           // If store file is not shared with other snapshots and tables,
302           // increase nonSharedHfilesArchiveSize
303           if ((filesMap != null) &&
304               !isArchivedFileStillReferenced(link.getArchivePath(), filesMap)) {
305             nonSharedHfilesArchiveSize.addAndGet(size);
306           }
307         } else if (inArchive = fs.exists(link.getMobPath())) {
308           size = fs.getFileStatus(link.getMobPath()).getLen();
309           hfilesMobSize.addAndGet(size);
310           hfilesMobCount.incrementAndGet();
311         } else {
312           size = link.getFileStatus(fs).getLen();
313           hfilesSize.addAndGet(size);
314           hfilesCount.incrementAndGet();
315         }
316         isCorrupted = (storeFile.hasFileSize() && storeFile.getFileSize() != size);
317         if (isCorrupted) hfilesCorrupted.incrementAndGet();
318       } catch (FileNotFoundException e) {
319         hfilesMissing.incrementAndGet();
320       }
321       return new FileInfo(inArchive, size, isCorrupted);
322     }
323
324     /**
325      * Add the specified log file to the stats
326      * @param server server name
327      * @param logfile log file name
328      * @return the log information
329      */
330     FileInfo addLogFile(final String server, final String logfile) throws IOException {
331       WALLink logLink = new WALLink(conf, server, logfile);
332       long size = -1;
333       try {
334         size = logLink.getFileStatus(fs).getLen();
335         logSize.addAndGet(size);
336         logsCount.incrementAndGet();
337       } catch (FileNotFoundException e) {
338         logsMissing.incrementAndGet();
339       }
340       return new FileInfo(false, size, false);
341     }
342   }
343
344   private boolean printSizeInBytes = false;
345   private FileSystem fs;
346   private Path rootDir;
347
348   private SnapshotManifest snapshotManifest;
349
350   @Override
351   @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="REC_CATCH_EXCEPTION",
352     justification="Intentional")
353   public int run(String[] args) throws IOException, InterruptedException {
354     final Configuration conf = getConf();
355     boolean listSnapshots = false;
356     String snapshotName = null;
357     boolean showSchema = false;
358     boolean showFiles = false;
359     boolean showStats = false;
360
361     // Process command line args
362     for (int i = 0; i < args.length; i++) {
363       String cmd = args[i];
364       try {
365         if (cmd.equals("-snapshot")) {
366           snapshotName = args[++i];
367         } else if (cmd.equals("-files")) {
368           showFiles = true;
369           showStats = true;
370         } else if (cmd.equals("-stats")) {
371           showStats = true;
372         } else if (cmd.equals("-schema")) {
373           showSchema = true;
374         } else if (cmd.equals("-remote-dir")) {
375           Path sourceDir = new Path(args[++i]);
376           URI defaultFs = sourceDir.getFileSystem(conf).getUri();
377           FSUtils.setFsDefault(conf, new Path(defaultFs));
378           FSUtils.setRootDir(conf, sourceDir);
379         } else if (cmd.equals("-list-snapshots")) {
380           listSnapshots = true;
381         } else if (cmd.equals("-size-in-bytes")) {
382           printSizeInBytes = true;
383         } else if (cmd.equals("-h") || cmd.equals("--help")) {
384           printUsageAndExit();
385         } else {
386           System.err.println("UNEXPECTED: " + cmd);
387           printUsageAndExit();
388         }
389       } catch (Exception e) {
390         printUsageAndExit(); // FindBugs: REC_CATCH_EXCEPTION
391       }
392     }
393
394     // List Available Snapshots
395     if (listSnapshots) {
396       SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
397       System.out.printf("%-20s | %-20s | %s%n", "SNAPSHOT", "CREATION TIME", "TABLE NAME");
398       for (SnapshotDescription desc: getSnapshotList(conf)) {
399         System.out.printf("%-20s | %20s | %s%n",
400                           desc.getName(),
401                           df.format(new Date(desc.getCreationTime())),
402                           desc.getTable());
403       }
404       return 0;
405     }
406
407     if (snapshotName == null) {
408       System.err.println("Missing snapshot name!");
409       printUsageAndExit();
410       return 1;
411     }
412
413     rootDir = FSUtils.getRootDir(conf);
414     fs = FileSystem.get(rootDir.toUri(), conf);
415     LOG.debug("fs=" + fs.getUri().toString() + " root=" + rootDir);
416
417     // Load snapshot information
418     if (!loadSnapshotInfo(snapshotName)) {
419       System.err.println("Snapshot '" + snapshotName + "' not found!");
420       return 1;
421     }
422
423     printInfo();
424     if (showSchema) printSchema();
425     printFiles(showFiles, showStats);
426
427     return 0;
428   }
429
430   /**
431    * Load snapshot info and table descriptor for the specified snapshot
432    * @param snapshotName name of the snapshot to load
433    * @return false if snapshot is not found
434    */
435   private boolean loadSnapshotInfo(final String snapshotName) throws IOException {
436     Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
437     if (!fs.exists(snapshotDir)) {
438       LOG.warn("Snapshot '" + snapshotName + "' not found in: " + snapshotDir);
439       return false;
440     }
441
442     HBaseProtos.SnapshotDescription snapshotDesc =
443         SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
444     snapshotManifest = SnapshotManifest.open(getConf(), fs, snapshotDir, snapshotDesc);
445     return true;
446   }
447
448   /**
449    * Dump the {@link SnapshotDescription}
450    */
451   private void printInfo() {
452     HBaseProtos.SnapshotDescription snapshotDesc = snapshotManifest.getSnapshotDescription();
453     SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
454     System.out.println("Snapshot Info");
455     System.out.println("----------------------------------------");
456     System.out.println("   Name: " + snapshotDesc.getName());
457     System.out.println("   Type: " + snapshotDesc.getType());
458     System.out.println("  Table: " + snapshotDesc.getTable());
459     System.out.println(" Format: " + snapshotDesc.getVersion());
460     System.out.println("Created: " + df.format(new Date(snapshotDesc.getCreationTime())));
461     System.out.println("  Owner: " + snapshotDesc.getOwner());
462     System.out.println();
463   }
464
465   /**
466    * Dump the {@link HTableDescriptor}
467    */
468   private void printSchema() {
469     System.out.println("Table Descriptor");
470     System.out.println("----------------------------------------");
471     System.out.println(snapshotManifest.getTableDescriptor().toString());
472     System.out.println();
473   }
474
475   /**
476    * Collect the hfiles and logs statistics of the snapshot and
477    * dump the file list if requested and the collected information.
478    */
479   private void printFiles(final boolean showFiles, final boolean showStats) throws IOException {
480     if (showFiles) {
481       System.out.println("Snapshot Files");
482       System.out.println("----------------------------------------");
483     }
484
485     // Collect information about hfiles and logs in the snapshot
486     final HBaseProtos.SnapshotDescription snapshotDesc = snapshotManifest.getSnapshotDescription();
487     final String table = snapshotDesc.getTable();
488     SnapshotDescription desc = new SnapshotDescription(snapshotDesc.getName(),
489         snapshotDesc.getTable(), ProtobufUtil.createSnapshotType(snapshotDesc.getType()),
490         snapshotDesc.getOwner(), snapshotDesc.getCreationTime(), snapshotDesc.getVersion());
491     final SnapshotStats stats = new SnapshotStats(this.getConf(), this.fs, desc);
492     SnapshotReferenceUtil.concurrentVisitReferencedFiles(getConf(), fs, snapshotManifest,
493         "SnapshotInfo",
494       new SnapshotReferenceUtil.SnapshotVisitor() {
495         @Override
496         public void storeFile(final HRegionInfo regionInfo, final String family,
497             final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
498           if (storeFile.hasReference()) return;
499
500           SnapshotStats.FileInfo info = stats.addStoreFile(regionInfo, family, storeFile, null);
501           if (showFiles) {
502             String state = info.getStateToString();
503             System.out.printf("%8s %s/%s/%s/%s %s%n",
504               (info.isMissing() ? "-" : fileSizeToString(info.getSize())),
505               table, regionInfo.getEncodedName(), family, storeFile.getName(),
506               state == null ? "" : "(" + state + ")");
507           }
508         }
509     });
510
511     // Dump the stats
512     System.out.println();
513     if (stats.isSnapshotCorrupted()) {
514       System.out.println("**************************************************************");
515       System.out.printf("BAD SNAPSHOT: %d hfile(s) and %d log(s) missing.%n",
516         stats.getMissingStoreFilesCount(), stats.getMissingLogsCount());
517       System.out.printf("              %d hfile(s) corrupted.%n",
518         stats.getCorruptedStoreFilesCount());
519       System.out.println("**************************************************************");
520     }
521
522     if (showStats) {
523       System.out.printf("%d HFiles (%d in archive, %d in mob storage), total size %s " +
524               "(%.2f%% %s shared with the source table, %.2f%% %s in mob dir)%n",
525         stats.getStoreFilesCount(), stats.getArchivedStoreFilesCount(),
526         stats.getMobStoreFilesCount(),
527         fileSizeToString(stats.getStoreFilesSize()),
528         stats.getSharedStoreFilePercentage(),
529         fileSizeToString(stats.getSharedStoreFilesSize()),
530         stats.getMobStoreFilePercentage(),
531         fileSizeToString(stats.getMobStoreFilesSize())
532       );
533       System.out.printf("%d Logs, total size %s%n",
534         stats.getLogsCount(), fileSizeToString(stats.getLogsSize()));
535       System.out.println();
536     }
537   }
538
539   private String fileSizeToString(long size) {
540     return printSizeInBytes ? Long.toString(size) : StringUtils.humanReadableInt(size);
541   }
542
543   private void printUsageAndExit() {
544     System.err.printf("Usage: bin/hbase snapshot info [options]%n");
545     System.err.println(" where [options] are:");
546     System.err.println("  -h|-help                Show this help and exit.");
547     System.err.println("  -remote-dir             Root directory that contains the snapshots.");
548     System.err.println("  -list-snapshots         List all the available snapshots and exit.");
549     System.err.println("  -size-in-bytes          Print the size of the files in bytes.");
550     System.err.println("  -snapshot NAME          Snapshot to examine.");
551     System.err.println("  -files                  Files and logs list.");
552     System.err.println("  -stats                  Files and logs stats.");
553     System.err.println("  -schema                 Describe the snapshotted table.");
554     System.err.println();
555     System.err.println("Examples:");
556     System.err.println("  hbase snapshot info \\");
557     System.err.println("    -snapshot MySnapshot -files");
558     System.exit(1);
559   }
560
561   /**
562    * Returns the snapshot stats
563    * @param conf the {@link Configuration} to use
564    * @param snapshot {@link SnapshotDescription} to get stats from
565    * @return the snapshot stats
566    */
567   public static SnapshotStats getSnapshotStats(final Configuration conf,
568       final SnapshotDescription snapshot) throws IOException {
569     HBaseProtos.SnapshotDescription snapshotDesc = ProtobufUtil.createHBaseProtosSnapshotDesc(
570         snapshot);
571
572     return getSnapshotStats(conf, snapshotDesc, null);
573   }
574
575   /**
576    * Returns the snapshot stats
577    * @param conf the {@link Configuration} to use
578    * @param snapshotDesc  HBaseProtos.SnapshotDescription to get stats from
579    * @param filesMap {@link Map} store files map for all snapshots, it may be null
580    * @return the snapshot stats
581    */
582   public static SnapshotStats getSnapshotStats(final Configuration conf,
583       final HBaseProtos.SnapshotDescription snapshotDesc,
584       final Map<Path, Integer> filesMap) throws IOException {
585     Path rootDir = FSUtils.getRootDir(conf);
586     FileSystem fs = FileSystem.get(rootDir.toUri(), conf);
587     Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotDesc, rootDir);
588     SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc);
589     final SnapshotStats stats = new SnapshotStats(conf, fs, snapshotDesc);
590     SnapshotReferenceUtil.concurrentVisitReferencedFiles(conf, fs, manifest,
591         "SnapshotsStatsAggregation", new SnapshotReferenceUtil.SnapshotVisitor() {
592           @Override
593           public void storeFile(final HRegionInfo regionInfo, final String family,
594               final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
595             if (!storeFile.hasReference()) {
596               stats.addStoreFile(regionInfo, family, storeFile, filesMap);
597             }
598           }});
599     return stats;
600   }
601
602   /**
603    * Returns the list of available snapshots in the specified location
604    * @param conf the {@link Configuration} to use
605    * @return the list of snapshots
606    */
607   public static List<SnapshotDescription> getSnapshotList(final Configuration conf)
608       throws IOException {
609     Path rootDir = FSUtils.getRootDir(conf);
610     FileSystem fs = FileSystem.get(rootDir.toUri(), conf);
611     Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
612     FileStatus[] snapshots = fs.listStatus(snapshotDir,
613         new SnapshotDescriptionUtils.CompletedSnaphotDirectoriesFilter(fs));
614     List<SnapshotDescription> snapshotLists =
615       new ArrayList<SnapshotDescription>(snapshots.length);
616     for (FileStatus snapshotDirStat: snapshots) {
617       HBaseProtos.SnapshotDescription snapshotDesc =
618           SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDirStat.getPath());
619       snapshotLists.add(new SnapshotDescription(snapshotDesc.getName(),
620           snapshotDesc.getTable(), ProtobufUtil.createSnapshotType(snapshotDesc.getType()),
621           snapshotDesc.getOwner(), snapshotDesc.getCreationTime(), snapshotDesc.getVersion()));
622     }
623     return snapshotLists;
624   }
625
626   /**
627    * Gets the store files map for snapshot
628    * @param conf the {@link Configuration} to use
629    * @param snapshot {@link SnapshotDescription} to get stats from
630    * @param exec the {@link ExecutorService} to use
631    * @param filesMap {@link Map} the map to put the mapping entries
632    * @param uniqueHFilesArchiveSize {@link AtomicLong} the accumulated store file size in archive
633    * @param uniqueHFilesSize {@link AtomicLong} the accumulated store file size shared
634    * @param uniqueHFilesMobSize {@link AtomicLong} the accumulated mob store file size shared
635    * @return the snapshot stats
636    */
637   private static void getSnapshotFilesMap(final Configuration conf,
638       final SnapshotDescription snapshot, final ExecutorService exec,
639       final ConcurrentHashMap<Path, Integer> filesMap,
640       final AtomicLong uniqueHFilesArchiveSize, final AtomicLong uniqueHFilesSize,
641       final AtomicLong uniqueHFilesMobSize) throws IOException {
642     HBaseProtos.SnapshotDescription snapshotDesc = ProtobufUtil.createHBaseProtosSnapshotDesc(
643         snapshot);
644     Path rootDir = FSUtils.getRootDir(conf);
645     final FileSystem fs = FileSystem.get(rootDir.toUri(), conf);
646
647     Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotDesc, rootDir);
648     SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc);
649     SnapshotReferenceUtil.concurrentVisitReferencedFiles(conf, fs, manifest, exec,
650         new SnapshotReferenceUtil.SnapshotVisitor() {
651           @Override public void storeFile(final HRegionInfo regionInfo, final String family,
652               final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
653             if (!storeFile.hasReference()) {
654               HFileLink link = HFileLink
655                   .build(conf, TableName.valueOf(snapshot.getTable()), regionInfo.getEncodedName(),
656                       family, storeFile.getName());
657               long size;
658               Integer count;
659               Path p;
660               AtomicLong al;
661               int c = 0;
662
663               if (fs.exists(link.getArchivePath())) {
664                 p = link.getArchivePath();
665                 al = uniqueHFilesArchiveSize;
666                 size = fs.getFileStatus(p).getLen();
667               } else if (fs.exists(link.getMobPath())) {
668                 p = link.getMobPath();
669                 al = uniqueHFilesMobSize;
670                 size = fs.getFileStatus(p).getLen();
671               } else {
672                 p = link.getOriginPath();
673                 al = uniqueHFilesSize;
674                 size = link.getFileStatus(fs).getLen();
675               }
676
677               // If it has been counted, do not double count
678               count = filesMap.get(p);
679               if (count != null) {
680                 c = count.intValue();
681               } else {
682                 al.addAndGet(size);
683               }
684
685               filesMap.put(p, ++c);
686             }
687           }
688         });
689   }
690
691   /**
692    * Returns the map of store files based on path for all snapshots
693    * @param conf the {@link Configuration} to use
694    * @param uniqueHFilesArchiveSize pass out the size for store files in archive
695    * @param uniqueHFilesSize pass out the size for store files shared
696    * @param uniqueHFilesMobSize pass out the size for mob store files shared
697    * @return the map of store files
698    */
699   public static Map<Path, Integer> getSnapshotsFilesMap(final Configuration conf,
700       AtomicLong uniqueHFilesArchiveSize, AtomicLong uniqueHFilesSize,
701       AtomicLong uniqueHFilesMobSize) throws IOException {
702     List<SnapshotDescription> snapshotList = getSnapshotList(conf);
703
704
705     if (snapshotList.size() == 0) {
706       return Collections.emptyMap();
707     }
708
709     ConcurrentHashMap<Path, Integer> fileMap = new ConcurrentHashMap<>();
710
711     ExecutorService exec = SnapshotManifest.createExecutor(conf, "SnapshotsFilesMapping");
712
713     try {
714       for (final SnapshotDescription snapshot : snapshotList) {
715         getSnapshotFilesMap(conf, snapshot, exec, fileMap, uniqueHFilesArchiveSize,
716             uniqueHFilesSize, uniqueHFilesMobSize);
717       }
718     } finally {
719       exec.shutdown();
720     }
721
722     return fileMap;
723   }
724
725   /**
726    * The guts of the {@link #main} method.
727    * Call this method to avoid the {@link #main(String[])} System.exit.
728    * @param args
729    * @return errCode
730    * @throws Exception
731    */
732   static int innerMain(final String [] args) throws Exception {
733     return ToolRunner.run(HBaseConfiguration.create(), new SnapshotInfo(), args);
734   }
735
736   public static void main(String[] args) throws Exception {
737      System.exit(innerMain(args));
738   }
739 }