1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.snapshot;
20
21 import java.io.IOException;
22 import java.io.FileNotFoundException;
23 import java.net.URI;
24 import java.text.SimpleDateFormat;
25 import java.util.ArrayList;
26 import java.util.Date;
27 import java.util.List;
28 import java.util.concurrent.atomic.AtomicInteger;
29 import java.util.concurrent.atomic.AtomicLong;
30
31 import org.apache.commons.logging.Log;
32 import org.apache.commons.logging.LogFactory;
33
34 import org.apache.hadoop.fs.Path;
35 import org.apache.hadoop.fs.FileStatus;
36 import org.apache.hadoop.fs.FileSystem;
37 import org.apache.hadoop.hbase.classification.InterfaceAudience;
38 import org.apache.hadoop.hbase.classification.InterfaceStability;
39 import org.apache.hadoop.conf.Configured;
40 import org.apache.hadoop.hbase.HRegionInfo;
41 import org.apache.hadoop.hbase.TableName;
42 import org.apache.hadoop.util.StringUtils;
43 import org.apache.hadoop.util.Tool;
44 import org.apache.hadoop.util.ToolRunner;
45
46 import org.apache.hadoop.conf.Configuration;
47 import org.apache.hadoop.hbase.HBaseConfiguration;
48 import org.apache.hadoop.hbase.io.HFileLink;
49 import org.apache.hadoop.hbase.io.WALLink;
50 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
51 import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
52 import org.apache.hadoop.hbase.util.FSUtils;
53
54
55
56
57
58
59
60
61
62
63 @InterfaceAudience.Public
64 @InterfaceStability.Evolving
65 public final class SnapshotInfo extends Configured implements Tool {
66 private static final Log LOG = LogFactory.getLog(SnapshotInfo.class);
67
68
69
70
71
72
73
74
75
76 public static class SnapshotStats {
77
78 static class FileInfo {
79 private final boolean corrupted;
80 private final boolean inArchive;
81 private final long size;
82
83 FileInfo(final boolean inArchive, final long size, final boolean corrupted) {
84 this.corrupted = corrupted;
85 this.inArchive = inArchive;
86 this.size = size;
87 }
88
89
90 public boolean inArchive() {
91 return this.inArchive;
92 }
93
94
95 public boolean isCorrupted() {
96 return this.corrupted;
97 }
98
99
100 public boolean isMissing() {
101 return this.size < 0;
102 }
103
104
105 public long getSize() {
106 return this.size;
107 }
108
109 String getStateToString() {
110 if (isCorrupted()) return "CORRUPTED";
111 if (isMissing()) return "NOT FOUND";
112 if (inArchive()) return "archive";
113 return null;
114 }
115 }
116
117 private AtomicInteger hfileArchiveCount = new AtomicInteger();
118 private AtomicInteger hfilesCorrupted = new AtomicInteger();
119 private AtomicInteger hfilesMissing = new AtomicInteger();
120 private AtomicInteger hfilesCount = new AtomicInteger();
121 private AtomicInteger logsMissing = new AtomicInteger();
122 private AtomicInteger logsCount = new AtomicInteger();
123 private AtomicLong hfileArchiveSize = new AtomicLong();
124 private AtomicLong hfileSize = new AtomicLong();
125 private AtomicLong logSize = new AtomicLong();
126
127 private final SnapshotDescription snapshot;
128 private final TableName snapshotTable;
129 private final Configuration conf;
130 private final FileSystem fs;
131
132 SnapshotStats(final Configuration conf, final FileSystem fs, final SnapshotDescription snapshot)
133 {
134 this.snapshot = snapshot;
135 this.snapshotTable = TableName.valueOf(snapshot.getTable());
136 this.conf = conf;
137 this.fs = fs;
138 }
139
140
141 public SnapshotDescription getSnapshotDescription() {
142 return this.snapshot;
143 }
144
145
146 public boolean isSnapshotCorrupted() {
147 return hfilesMissing.get() > 0 ||
148 logsMissing.get() > 0 ||
149 hfilesCorrupted.get() > 0;
150 }
151
152
153 public int getStoreFilesCount() {
154 return hfilesCount.get() + hfileArchiveCount.get();
155 }
156
157
158 public int getArchivedStoreFilesCount() {
159 return hfileArchiveCount.get();
160 }
161
162
163 public int getLogsCount() {
164 return logsCount.get();
165 }
166
167
168 public int getMissingStoreFilesCount() {
169 return hfilesMissing.get();
170 }
171
172
173 public int getCorruptedStoreFilesCount() {
174 return hfilesCorrupted.get();
175 }
176
177
178 public int getMissingLogsCount() {
179 return logsMissing.get();
180 }
181
182
183 public long getStoreFilesSize() {
184 return hfileSize.get() + hfileArchiveSize.get();
185 }
186
187
188 public long getSharedStoreFilesSize() {
189 return hfileSize.get();
190 }
191
192
193 public long getArchivedStoreFileSize() {
194 return hfileArchiveSize.get();
195 }
196
197
198 public float getSharedStoreFilePercentage() {
199 return ((float)hfileSize.get() / (hfileSize.get() + hfileArchiveSize.get())) * 100;
200 }
201
202
203 public long getLogsSize() {
204 return logSize.get();
205 }
206
207
208
209
210
211
212
213
214 FileInfo addStoreFile(final HRegionInfo region, final String family,
215 final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
216 HFileLink link = HFileLink.build(conf, snapshotTable, region.getEncodedName(),
217 family, storeFile.getName());
218 boolean isCorrupted = false;
219 boolean inArchive = false;
220 long size = -1;
221 try {
222 if ((inArchive = fs.exists(link.getArchivePath()))) {
223 size = fs.getFileStatus(link.getArchivePath()).getLen();
224 hfileArchiveSize.addAndGet(size);
225 hfileArchiveCount.incrementAndGet();
226 } else {
227 size = link.getFileStatus(fs).getLen();
228 hfileSize.addAndGet(size);
229 hfilesCount.incrementAndGet();
230 }
231 isCorrupted = (storeFile.hasFileSize() && storeFile.getFileSize() != size);
232 if (isCorrupted) hfilesCorrupted.incrementAndGet();
233 } catch (FileNotFoundException e) {
234 hfilesMissing.incrementAndGet();
235 }
236 return new FileInfo(inArchive, size, isCorrupted);
237 }
238
239
240
241
242
243
244
245 FileInfo addLogFile(final String server, final String logfile) throws IOException {
246 WALLink logLink = new WALLink(conf, server, logfile);
247 long size = -1;
248 try {
249 size = logLink.getFileStatus(fs).getLen();
250 logSize.addAndGet(size);
251 logsCount.incrementAndGet();
252 } catch (FileNotFoundException e) {
253 logsMissing.incrementAndGet();
254 }
255 return new FileInfo(false, size, false);
256 }
257 }
258
259 private boolean printSizeInBytes = false;
260 private FileSystem fs;
261 private Path rootDir;
262
263 private SnapshotManifest snapshotManifest;
264
265 @Override
266 @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="REC_CATCH_EXCEPTION",
267 justification="Intentional")
268 public int run(String[] args) throws IOException, InterruptedException {
269 final Configuration conf = getConf();
270 boolean listSnapshots = false;
271 String snapshotName = null;
272 boolean showSchema = false;
273 boolean showFiles = false;
274 boolean showStats = false;
275
276
277 for (int i = 0; i < args.length; i++) {
278 String cmd = args[i];
279 try {
280 if (cmd.equals("-snapshot")) {
281 snapshotName = args[++i];
282 } else if (cmd.equals("-files")) {
283 showFiles = true;
284 showStats = true;
285 } else if (cmd.equals("-stats")) {
286 showStats = true;
287 } else if (cmd.equals("-schema")) {
288 showSchema = true;
289 } else if (cmd.equals("-remote-dir")) {
290 Path sourceDir = new Path(args[++i]);
291 URI defaultFs = sourceDir.getFileSystem(conf).getUri();
292 FSUtils.setFsDefault(conf, new Path(defaultFs));
293 FSUtils.setRootDir(conf, sourceDir);
294 } else if (cmd.equals("-list-snapshots")) {
295 listSnapshots = true;
296 } else if (cmd.equals("-size-in-bytes")) {
297 printSizeInBytes = true;
298 } else if (cmd.equals("-h") || cmd.equals("--help")) {
299 printUsageAndExit();
300 } else {
301 System.err.println("UNEXPECTED: " + cmd);
302 printUsageAndExit();
303 }
304 } catch (Exception e) {
305 printUsageAndExit();
306 }
307 }
308
309
310 if (listSnapshots) {
311 SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
312 System.out.printf("%-20s | %-20s | %s%n", "SNAPSHOT", "CREATION TIME", "TABLE NAME");
313 for (SnapshotDescription desc: getSnapshotList(conf)) {
314 System.out.printf("%-20s | %20s | %s%n",
315 desc.getName(),
316 df.format(new Date(desc.getCreationTime())),
317 desc.getTable());
318 }
319 return 0;
320 }
321
322 if (snapshotName == null) {
323 System.err.println("Missing snapshot name!");
324 printUsageAndExit();
325 return 1;
326 }
327
328 rootDir = FSUtils.getRootDir(conf);
329 fs = FileSystem.get(rootDir.toUri(), conf);
330 LOG.debug("fs=" + fs.getUri().toString() + " root=" + rootDir);
331
332
333 if (!loadSnapshotInfo(snapshotName)) {
334 System.err.println("Snapshot '" + snapshotName + "' not found!");
335 return 1;
336 }
337
338 printInfo();
339 if (showSchema) printSchema();
340 printFiles(showFiles, showStats);
341
342 return 0;
343 }
344
345
346
347
348
349
350 private boolean loadSnapshotInfo(final String snapshotName) throws IOException {
351 Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
352 if (!fs.exists(snapshotDir)) {
353 LOG.warn("Snapshot '" + snapshotName + "' not found in: " + snapshotDir);
354 return false;
355 }
356
357 SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
358 snapshotManifest = SnapshotManifest.open(getConf(), fs, snapshotDir, snapshotDesc);
359 return true;
360 }
361
362
363
364
365 private void printInfo() {
366 SnapshotDescription snapshotDesc = snapshotManifest.getSnapshotDescription();
367 SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
368 System.out.println("Snapshot Info");
369 System.out.println("----------------------------------------");
370 System.out.println(" Name: " + snapshotDesc.getName());
371 System.out.println(" Type: " + snapshotDesc.getType());
372 System.out.println(" Table: " + snapshotDesc.getTable());
373 System.out.println(" Format: " + snapshotDesc.getVersion());
374 System.out.println("Created: " + df.format(new Date(snapshotDesc.getCreationTime())));
375 System.out.println();
376 }
377
378
379
380
381 private void printSchema() {
382 System.out.println("Table Descriptor");
383 System.out.println("----------------------------------------");
384 System.out.println(snapshotManifest.getTableDescriptor().toString());
385 System.out.println();
386 }
387
388
389
390
391
392 private void printFiles(final boolean showFiles, final boolean showStats) throws IOException {
393 if (showFiles) {
394 System.out.println("Snapshot Files");
395 System.out.println("----------------------------------------");
396 }
397
398
399 final SnapshotDescription snapshotDesc = snapshotManifest.getSnapshotDescription();
400 final String table = snapshotDesc.getTable();
401 final SnapshotStats stats = new SnapshotStats(this.getConf(), this.fs, snapshotDesc);
402 SnapshotReferenceUtil.concurrentVisitReferencedFiles(getConf(), fs, snapshotManifest,
403 new SnapshotReferenceUtil.SnapshotVisitor() {
404 @Override
405 public void storeFile(final HRegionInfo regionInfo, final String family,
406 final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
407 if (storeFile.hasReference()) return;
408
409 SnapshotStats.FileInfo info = stats.addStoreFile(regionInfo, family, storeFile);
410 if (showFiles) {
411 String state = info.getStateToString();
412 System.out.printf("%8s %s/%s/%s/%s %s%n",
413 (info.isMissing() ? "-" : fileSizeToString(info.getSize())),
414 table, regionInfo.getEncodedName(), family, storeFile.getName(),
415 state == null ? "" : "(" + state + ")");
416 }
417 }
418
419 @Override
420 public void logFile (final String server, final String logfile)
421 throws IOException {
422 SnapshotStats.FileInfo info = stats.addLogFile(server, logfile);
423
424 if (showFiles) {
425 String state = info.getStateToString();
426 System.out.printf("%8s log %s on server %s (%s)%n",
427 (info.isMissing() ? "-" : fileSizeToString(info.getSize())),
428 logfile, server,
429 state == null ? "" : "(" + state + ")");
430 }
431 }
432 });
433
434
435 System.out.println();
436 if (stats.isSnapshotCorrupted()) {
437 System.out.println("**************************************************************");
438 System.out.printf("BAD SNAPSHOT: %d hfile(s) and %d log(s) missing.%n",
439 stats.getMissingStoreFilesCount(), stats.getMissingLogsCount());
440 System.out.printf(" %d hfile(s) corrupted.%n",
441 stats.getCorruptedStoreFilesCount());
442 System.out.println("**************************************************************");
443 }
444
445 if (showStats) {
446 System.out.printf("%d HFiles (%d in archive), total size %s (%.2f%% %s shared with the source table)%n",
447 stats.getStoreFilesCount(), stats.getArchivedStoreFilesCount(),
448 fileSizeToString(stats.getStoreFilesSize()),
449 stats.getSharedStoreFilePercentage(),
450 fileSizeToString(stats.getSharedStoreFilesSize())
451 );
452 System.out.printf("%d Logs, total size %s%n",
453 stats.getLogsCount(), fileSizeToString(stats.getLogsSize()));
454 System.out.println();
455 }
456 }
457
458 private String fileSizeToString(long size) {
459 return printSizeInBytes ? Long.toString(size) : StringUtils.humanReadableInt(size);
460 }
461
462 private void printUsageAndExit() {
463 System.err.printf("Usage: bin/hbase %s [options]%n", getClass().getName());
464 System.err.println(" where [options] are:");
465 System.err.println(" -h|-help Show this help and exit.");
466 System.err.println(" -remote-dir Root directory that contains the snapshots.");
467 System.err.println(" -list-snapshots List all the available snapshots and exit.");
468 System.err.println(" -size-in-bytes Print the size of the files in bytes.");
469 System.err.println(" -snapshot NAME Snapshot to examine.");
470 System.err.println(" -files Files and logs list.");
471 System.err.println(" -stats Files and logs stats.");
472 System.err.println(" -schema Describe the snapshotted table.");
473 System.err.println();
474 System.err.println("Examples:");
475 System.err.println(" hbase " + getClass() + " \\");
476 System.err.println(" -snapshot MySnapshot -files");
477 System.exit(1);
478 }
479
480
481
482
483
484
485
486 public static SnapshotStats getSnapshotStats(final Configuration conf,
487 final SnapshotDescription snapshot) throws IOException {
488 Path rootDir = FSUtils.getRootDir(conf);
489 FileSystem fs = FileSystem.get(rootDir.toUri(), conf);
490 Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
491 SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshot);
492 final SnapshotStats stats = new SnapshotStats(conf, fs, snapshot);
493 SnapshotReferenceUtil.concurrentVisitReferencedFiles(conf, fs, manifest,
494 new SnapshotReferenceUtil.SnapshotVisitor() {
495 @Override
496 public void storeFile(final HRegionInfo regionInfo, final String family,
497 final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
498 if (!storeFile.hasReference()) {
499 stats.addStoreFile(regionInfo, family, storeFile);
500 }
501 }
502
503 @Override
504 public void logFile (final String server, final String logfile) throws IOException {
505 stats.addLogFile(server, logfile);
506 }
507 });
508 return stats;
509 }
510
511
512
513
514
515
516 public static List<SnapshotDescription> getSnapshotList(final Configuration conf)
517 throws IOException {
518 Path rootDir = FSUtils.getRootDir(conf);
519 FileSystem fs = FileSystem.get(rootDir.toUri(), conf);
520 Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
521 FileStatus[] snapshots = fs.listStatus(snapshotDir,
522 new SnapshotDescriptionUtils.CompletedSnaphotDirectoriesFilter(fs));
523 List<SnapshotDescription> snapshotLists =
524 new ArrayList<SnapshotDescription>(snapshots.length);
525 for (FileStatus snapshotDirStat: snapshots) {
526 snapshotLists.add(SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDirStat.getPath()));
527 }
528 return snapshotLists;
529 }
530
531
532
533
534
535
536
537
538 static int innerMain(final String [] args) throws Exception {
539 return ToolRunner.run(HBaseConfiguration.create(), new SnapshotInfo(), args);
540 }
541
542 public static void main(String[] args) throws Exception {
543 System.exit(innerMain(args));
544 }
545 }