001/*
002 *
003 * Licensed to the Apache Software Foundation (ASF) under one
004 * or more contributor license agreements.  See the NOTICE file
005 * distributed with this work for additional information
006 * regarding copyright ownership.  The ASF licenses this file
007 * to you under the Apache License, Version 2.0 (the
008 * "License"); you may not use this file except in compliance
009 * with the License.  You may obtain a copy of the License at
010 *
011 *     http://www.apache.org/licenses/LICENSE-2.0
012 *
013 * Unless required by applicable law or agreed to in writing, software
014 * distributed under the License is distributed on an "AS IS" BASIS,
015 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
016 * See the License for the specific language governing permissions and
017 * limitations under the License.
018 */
019package org.apache.hadoop.hbase.io.hfile;
020
021import static com.codahale.metrics.MetricRegistry.name;
022
023import com.codahale.metrics.ConsoleReporter;
024import com.codahale.metrics.Counter;
025import com.codahale.metrics.Gauge;
026import com.codahale.metrics.Histogram;
027import com.codahale.metrics.Meter;
028import com.codahale.metrics.MetricFilter;
029import com.codahale.metrics.MetricRegistry;
030import com.codahale.metrics.ScheduledReporter;
031import com.codahale.metrics.Snapshot;
032import com.codahale.metrics.Timer;
033import java.io.ByteArrayOutputStream;
034import java.io.DataInput;
035import java.io.IOException;
036import java.io.PrintStream;
037import java.text.DateFormat;
038import java.util.ArrayList;
039import java.util.HashMap;
040import java.util.Iterator;
041import java.util.LinkedHashSet;
042import java.util.List;
043import java.util.Locale;
044import java.util.Map;
045import java.util.Optional;
046import java.util.Set;
047import java.util.SortedMap;
048import java.util.TimeZone;
049import java.util.concurrent.TimeUnit;
050import org.apache.commons.lang3.StringUtils;
051import org.apache.hadoop.conf.Configuration;
052import org.apache.hadoop.conf.Configured;
053import org.apache.hadoop.fs.FileSystem;
054import org.apache.hadoop.fs.Path;
055import org.apache.hadoop.hbase.Cell;
056import org.apache.hadoop.hbase.CellComparator;
057import org.apache.hadoop.hbase.CellUtil;
058import org.apache.hadoop.hbase.HBaseConfiguration;
059import org.apache.hadoop.hbase.HBaseInterfaceAudience;
060import org.apache.hadoop.hbase.HConstants;
061import org.apache.hadoop.hbase.KeyValue;
062import org.apache.hadoop.hbase.KeyValueUtil;
063import org.apache.hadoop.hbase.PrivateCellUtil;
064import org.apache.hadoop.hbase.TableName;
065import org.apache.hadoop.hbase.Tag;
066import org.apache.hadoop.hbase.client.RegionInfo;
067import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper;
068import org.apache.hadoop.hbase.mob.MobUtils;
069import org.apache.hadoop.hbase.regionserver.HStoreFile;
070import org.apache.hadoop.hbase.regionserver.TimeRangeTracker;
071import org.apache.hadoop.hbase.util.BloomFilter;
072import org.apache.hadoop.hbase.util.BloomFilterFactory;
073import org.apache.hadoop.hbase.util.BloomFilterUtil;
074import org.apache.hadoop.hbase.util.Bytes;
075import org.apache.hadoop.hbase.util.CommonFSUtils;
076import org.apache.hadoop.hbase.util.HFileArchiveUtil;
077import org.apache.hadoop.util.Tool;
078import org.apache.hadoop.util.ToolRunner;
079import org.apache.yetus.audience.InterfaceAudience;
080import org.apache.yetus.audience.InterfaceStability;
081import org.slf4j.Logger;
082import org.slf4j.LoggerFactory;
083
084import org.apache.hbase.thirdparty.org.apache.commons.cli.CommandLine;
085import org.apache.hbase.thirdparty.org.apache.commons.cli.CommandLineParser;
086import org.apache.hbase.thirdparty.org.apache.commons.cli.HelpFormatter;
087import org.apache.hbase.thirdparty.org.apache.commons.cli.Option;
088import org.apache.hbase.thirdparty.org.apache.commons.cli.OptionGroup;
089import org.apache.hbase.thirdparty.org.apache.commons.cli.Options;
090import org.apache.hbase.thirdparty.org.apache.commons.cli.ParseException;
091import org.apache.hbase.thirdparty.org.apache.commons.cli.PosixParser;
092
093/**
094 * Implements pretty-printing functionality for {@link HFile}s.
095 */
096@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.TOOLS)
097@InterfaceStability.Evolving
098public class HFilePrettyPrinter extends Configured implements Tool {
099
100  private static final Logger LOG = LoggerFactory.getLogger(HFilePrettyPrinter.class);
101
102  private Options options = new Options();
103
104  private boolean verbose;
105  private boolean printValue;
106  private boolean printKey;
107  private boolean shouldPrintMeta;
108  private boolean printBlockIndex;
109  private boolean printBlockHeaders;
110  private boolean printStats;
111  private boolean checkRow;
112  private boolean checkFamily;
113  private boolean isSeekToRow = false;
114  private boolean checkMobIntegrity = false;
115  private Map<String, List<Path>> mobFileLocations;
116  private static final int FOUND_MOB_FILES_CACHE_CAPACITY = 50;
117  private static final int MISSING_MOB_FILES_CACHE_CAPACITY = 20;
118  private PrintStream out = System.out;
119  private PrintStream err = System.err;
120
121  /**
122   * The row which the user wants to specify and print all the KeyValues for.
123   */
124  private byte[] row = null;
125
126  private List<Path> files = new ArrayList<>();
127  private int count;
128
129  private static final String FOUR_SPACES = "    ";
130
131  public HFilePrettyPrinter() {
132    super();
133    init();
134  }
135
136  public HFilePrettyPrinter(Configuration conf) {
137    super(conf);
138    init();
139  }
140
141  private void init() {
142    options.addOption("v", "verbose", false,
143        "Verbose output; emits file and meta data delimiters");
144    options.addOption("p", "printkv", false, "Print key/value pairs");
145    options.addOption("e", "printkey", false, "Print keys");
146    options.addOption("m", "printmeta", false, "Print meta data of file");
147    options.addOption("b", "printblocks", false, "Print block index meta data");
148    options.addOption("h", "printblockheaders", false, "Print block headers for each block.");
149    options.addOption("k", "checkrow", false,
150        "Enable row order check; looks for out-of-order keys");
151    options.addOption("a", "checkfamily", false, "Enable family check");
152    options.addOption("w", "seekToRow", true,
153      "Seek to this row and print all the kvs for this row only");
154    options.addOption("s", "stats", false, "Print statistics");
155    options.addOption("i", "checkMobIntegrity", false,
156      "Print all cells whose mob files are missing");
157
158    OptionGroup files = new OptionGroup();
159    files.addOption(new Option("f", "file", true,
160      "File to scan. Pass full-path; e.g. hdfs://a:9000/hbase/hbase:meta/12/34"));
161    files.addOption(new Option("r", "region", true,
162      "Region to scan. Pass region name; e.g. 'hbase:meta,,1'"));
163    options.addOptionGroup(files);
164  }
165
166  public void setPrintStreams(PrintStream out, PrintStream err) {
167    this.out = out;
168    this.err = err;
169  }
170
171  public boolean parseOptions(String args[]) throws ParseException,
172      IOException {
173    if (args.length == 0) {
174      HelpFormatter formatter = new HelpFormatter();
175      formatter.printHelp("hfile", options, true);
176      return false;
177    }
178    CommandLineParser parser = new PosixParser();
179    CommandLine cmd = parser.parse(options, args);
180
181    verbose = cmd.hasOption("v");
182    printValue = cmd.hasOption("p");
183    printKey = cmd.hasOption("e") || printValue;
184    shouldPrintMeta = cmd.hasOption("m");
185    printBlockIndex = cmd.hasOption("b");
186    printBlockHeaders = cmd.hasOption("h");
187    printStats = cmd.hasOption("s");
188    checkRow = cmd.hasOption("k");
189    checkFamily = cmd.hasOption("a");
190    checkMobIntegrity = cmd.hasOption("i");
191
192    if (cmd.hasOption("f")) {
193      files.add(new Path(cmd.getOptionValue("f")));
194    }
195
196    if (cmd.hasOption("w")) {
197      String key = cmd.getOptionValue("w");
198      if (key != null && key.length() != 0) {
199        row = Bytes.toBytesBinary(key);
200        isSeekToRow = true;
201      } else {
202        err.println("Invalid row is specified.");
203        System.exit(-1);
204      }
205    }
206
207    if (cmd.hasOption("r")) {
208      String regionName = cmd.getOptionValue("r");
209      byte[] rn = Bytes.toBytes(regionName);
210      byte[][] hri = RegionInfo.parseRegionName(rn);
211      Path rootDir = CommonFSUtils.getRootDir(getConf());
212      Path tableDir = CommonFSUtils.getTableDir(rootDir, TableName.valueOf(hri[0]));
213      String enc = RegionInfo.encodeRegionName(rn);
214      Path regionDir = new Path(tableDir, enc);
215      if (verbose)
216        out.println("region dir -> " + regionDir);
217      List<Path> regionFiles = HFile.getStoreFiles(FileSystem.get(getConf()),
218          regionDir);
219      if (verbose)
220        out.println("Number of region files found -> "
221            + regionFiles.size());
222      if (verbose) {
223        int i = 1;
224        for (Path p : regionFiles) {
225          if (verbose)
226            out.println("Found file[" + i++ + "] -> " + p);
227        }
228      }
229      files.addAll(regionFiles);
230    }
231
232    if(checkMobIntegrity) {
233      if (verbose) {
234        System.out.println("checkMobIntegrity is enabled");
235      }
236      mobFileLocations = new HashMap<>();
237    }
238
239    cmd.getArgList().forEach((file) -> files.add(new Path(file)));
240
241    return true;
242  }
243
244  /**
245   * Runs the command-line pretty-printer, and returns the desired command
246   * exit code (zero for success, non-zero for failure).
247   */
248  @Override
249  public int run(String[] args) {
250    if (getConf() == null) {
251      throw new RuntimeException("A Configuration instance must be provided.");
252    }
253    try {
254      CommonFSUtils.setFsDefault(getConf(), CommonFSUtils.getRootDir(getConf()));
255      if (!parseOptions(args)) {
256        return 1;
257      }
258    } catch (IOException ex) {
259      LOG.error("Error parsing command-line options", ex);
260      return 1;
261    } catch (ParseException ex) {
262      LOG.error("Error parsing command-line options", ex);
263      return 1;
264    }
265
266    // iterate over all files found
267    for (Path fileName : files) {
268      try {
269        int exitCode = processFile(fileName, false);
270        if (exitCode != 0) {
271          return exitCode;
272        }
273      } catch (IOException ex) {
274        LOG.error("Error reading " + fileName, ex);
275        return -2;
276      }
277    }
278
279    if (verbose || printKey) {
280      out.println("Scanned kv count -> " + count);
281    }
282
283    return 0;
284  }
285
286  // HBASE-22561 introduces boolean checkRootDir for WebUI specificly
287  public int processFile(Path file, boolean checkRootDir) throws IOException {
288    if (verbose) {
289      out.println("Scanning -> " + file);
290    }
291
292    if (checkRootDir) {
293      Path rootPath = CommonFSUtils.getRootDir(getConf());
294      String rootString = rootPath + rootPath.SEPARATOR;
295      if (!file.toString().startsWith(rootString)) {
296        // First we see if fully-qualified URI matches the root dir. It might
297        // also be an absolute path in the same filesystem, so we prepend the FS
298        // of the root dir and see if that fully-qualified URI matches.
299        FileSystem rootFS = rootPath.getFileSystem(getConf());
300        String qualifiedFile = rootFS.getUri().toString() + file.toString();
301        if (!qualifiedFile.startsWith(rootString)) {
302          err.println(
303            "ERROR, file (" + file + ") is not in HBase's root directory (" + rootString + ")");
304          return -2;
305        }
306      }
307    }
308
309    FileSystem fs = file.getFileSystem(getConf());
310    if (!fs.exists(file)) {
311      err.println("ERROR, file doesnt exist: " + file);
312      return -2;
313    }
314
315    HFile.Reader reader = HFile.createReader(fs, file, CacheConfig.DISABLED, true, getConf());
316
317    Map<byte[], byte[]> fileInfo = reader.getHFileInfo();
318
319    KeyValueStatsCollector fileStats = null;
320
321    if (verbose || printKey || checkRow || checkFamily || printStats || checkMobIntegrity) {
322      // scan over file and read key/value's and check if requested
323      HFileScanner scanner = reader.getScanner(false, false, false);
324      fileStats = new KeyValueStatsCollector();
325      boolean shouldScanKeysValues = false;
326      if (this.isSeekToRow) {
327        // seek to the first kv on this row
328        shouldScanKeysValues =
329          (scanner.seekTo(PrivateCellUtil.createFirstOnRow(this.row)) != -1);
330      } else {
331        shouldScanKeysValues = scanner.seekTo();
332      }
333      if (shouldScanKeysValues)
334        scanKeysValues(file, fileStats, scanner, row);
335    }
336
337    // print meta data
338    if (shouldPrintMeta) {
339      printMeta(reader, fileInfo);
340    }
341
342    if (printBlockIndex) {
343      out.println("Block Index:");
344      out.println(reader.getDataBlockIndexReader());
345    }
346
347    if (printBlockHeaders) {
348      out.println("Block Headers:");
349      /*
350       * TODO: this same/similar block iteration logic is used in HFileBlock#blockRange and
351       * TestLazyDataBlockDecompression. Refactor?
352       */
353      FSDataInputStreamWrapper fsdis = new FSDataInputStreamWrapper(fs, file);
354      long fileSize = fs.getFileStatus(file).getLen();
355      FixedFileTrailer trailer =
356        FixedFileTrailer.readFromStream(fsdis.getStream(false), fileSize);
357      long offset = trailer.getFirstDataBlockOffset(),
358        max = trailer.getLastDataBlockOffset();
359      HFileBlock block;
360      while (offset <= max) {
361        block = reader.readBlock(offset, -1, /* cacheBlock */ false, /* pread */ false,
362          /* isCompaction */ false, /* updateCacheMetrics */ false, null, null);
363        offset += block.getOnDiskSizeWithHeader();
364        out.println(block);
365      }
366    }
367
368    if (printStats) {
369      fileStats.finish();
370      out.println("Stats:\n" + fileStats);
371    }
372
373    reader.close();
374    return 0;
375  }
376
377  private void scanKeysValues(Path file, KeyValueStatsCollector fileStats,
378      HFileScanner scanner,  byte[] row) throws IOException {
379    Cell pCell = null;
380    FileSystem fs = FileSystem.get(getConf());
381    Set<String> foundMobFiles = new LinkedHashSet<>(FOUND_MOB_FILES_CACHE_CAPACITY);
382    Set<String> missingMobFiles = new LinkedHashSet<>(MISSING_MOB_FILES_CACHE_CAPACITY);
383    do {
384      Cell cell = scanner.getCell();
385      if (row != null && row.length != 0) {
386        int result = CellComparator.getInstance().compareRows(cell, row, 0, row.length);
387        if (result > 0) {
388          break;
389        } else if (result < 0) {
390          continue;
391        }
392      }
393      // collect stats
394      if (printStats) {
395        fileStats.collect(cell);
396      }
397      // dump key value
398      if (printKey) {
399        out.print("K: " + cell);
400        if (printValue) {
401          out.print(" V: "
402              + Bytes.toStringBinary(cell.getValueArray(), cell.getValueOffset(),
403                  cell.getValueLength()));
404          int i = 0;
405          List<Tag> tags = PrivateCellUtil.getTags(cell);
406          for (Tag tag : tags) {
407            out.print(String.format(" T[%d]: %s", i++, tag.toString()));
408          }
409        }
410        out.println();
411      }
412      // check if rows are in order
413      if (checkRow && pCell != null) {
414        if (CellComparator.getInstance().compareRows(pCell, cell) > 0) {
415          err.println("WARNING, previous row is greater then"
416              + " current row\n\tfilename -> " + file + "\n\tprevious -> "
417              + CellUtil.getCellKeyAsString(pCell) + "\n\tcurrent  -> "
418              + CellUtil.getCellKeyAsString(cell));
419        }
420      }
421      // check if families are consistent
422      if (checkFamily) {
423        String fam = Bytes.toString(cell.getFamilyArray(), cell.getFamilyOffset(),
424            cell.getFamilyLength());
425        if (!file.toString().contains(fam)) {
426          err.println("WARNING, filename does not match kv family,"
427              + "\n\tfilename -> " + file + "\n\tkeyvalue -> "
428              + CellUtil.getCellKeyAsString(cell));
429        }
430        if (pCell != null && CellComparator.getInstance().compareFamilies(pCell, cell) != 0) {
431          err.println("WARNING, previous kv has different family"
432              + " compared to current key\n\tfilename -> " + file
433              + "\n\tprevious -> " + CellUtil.getCellKeyAsString(pCell)
434              + "\n\tcurrent  -> " + CellUtil.getCellKeyAsString(cell));
435        }
436      }
437      // check if mob files are missing.
438      if (checkMobIntegrity && MobUtils.isMobReferenceCell(cell)) {
439        Optional<TableName> tn = MobUtils.getTableName(cell);
440        if (! tn.isPresent()) {
441          System.err.println("ERROR, wrong tag format in mob reference cell "
442            + CellUtil.getCellKeyAsString(cell));
443        } else if (!MobUtils.hasValidMobRefCellValue(cell)) {
444          System.err.println("ERROR, wrong value format in mob reference cell "
445            + CellUtil.getCellKeyAsString(cell));
446        } else {
447          String mobFileName = MobUtils.getMobFileName(cell);
448          boolean exist = mobFileExists(fs, tn.get(), mobFileName,
449            Bytes.toString(CellUtil.cloneFamily(cell)), foundMobFiles, missingMobFiles);
450          if (!exist) {
451            // report error
452            System.err.println("ERROR, the mob file [" + mobFileName
453              + "] is missing referenced by cell " + CellUtil.getCellKeyAsString(cell));
454          }
455        }
456      }
457      pCell = cell;
458      ++count;
459    } while (scanner.next());
460  }
461
462  /**
463   * Checks whether the referenced mob file exists.
464   */
465  private boolean mobFileExists(FileSystem fs, TableName tn, String mobFileName, String family,
466    Set<String> foundMobFiles, Set<String> missingMobFiles) throws IOException {
467    if (foundMobFiles.contains(mobFileName)) {
468      return true;
469    }
470    if (missingMobFiles.contains(mobFileName)) {
471      return false;
472    }
473    String tableName = tn.getNameAsString();
474    List<Path> locations = mobFileLocations.get(tableName);
475    if (locations == null) {
476      locations = new ArrayList<>(2);
477      locations.add(MobUtils.getMobFamilyPath(getConf(), tn, family));
478      locations.add(HFileArchiveUtil.getStoreArchivePath(getConf(), tn,
479        MobUtils.getMobRegionInfo(tn).getEncodedName(), family));
480      mobFileLocations.put(tn.getNameAsString(), locations);
481    }
482    boolean exist = false;
483    for (Path location : locations) {
484      Path mobFilePath = new Path(location, mobFileName);
485      if (fs.exists(mobFilePath)) {
486        exist = true;
487        break;
488      }
489    }
490    if (exist) {
491      evictMobFilesIfNecessary(foundMobFiles, FOUND_MOB_FILES_CACHE_CAPACITY);
492      foundMobFiles.add(mobFileName);
493    } else {
494      evictMobFilesIfNecessary(missingMobFiles, MISSING_MOB_FILES_CACHE_CAPACITY);
495      missingMobFiles.add(mobFileName);
496    }
497    return exist;
498  }
499
500  /**
501   * Evicts the cached mob files if the set is larger than the limit.
502   */
503  private void evictMobFilesIfNecessary(Set<String> mobFileNames, int limit) {
504    if (mobFileNames.size() < limit) {
505      return;
506    }
507    int index = 0;
508    int evict = limit / 2;
509    Iterator<String> fileNamesItr = mobFileNames.iterator();
510    while (index < evict && fileNamesItr.hasNext()) {
511      fileNamesItr.next();
512      fileNamesItr.remove();
513      index++;
514    }
515  }
516
517  /**
518   * Format a string of the form "k1=v1, k2=v2, ..." into separate lines
519   * with a four-space indentation.
520   */
521  private static String asSeparateLines(String keyValueStr) {
522    return keyValueStr.replaceAll(", ([a-zA-Z]+=)",
523                                  ",\n" + FOUR_SPACES + "$1");
524  }
525
526  private void printMeta(HFile.Reader reader, Map<byte[], byte[]> fileInfo)
527      throws IOException {
528    out.println("Block index size as per heapsize: "
529        + reader.indexSize());
530    out.println(asSeparateLines(reader.toString()));
531    out.println("Trailer:\n    "
532        + asSeparateLines(reader.getTrailer().toString()));
533    out.println("Fileinfo:");
534    for (Map.Entry<byte[], byte[]> e : fileInfo.entrySet()) {
535      out.print(FOUR_SPACES + Bytes.toString(e.getKey()) + " = ");
536      if (Bytes.equals(e.getKey(), HStoreFile.MAX_SEQ_ID_KEY)
537          || Bytes.equals(e.getKey(), HStoreFile.DELETE_FAMILY_COUNT)
538          || Bytes.equals(e.getKey(), HStoreFile.EARLIEST_PUT_TS)
539          || Bytes.equals(e.getKey(), HFileWriterImpl.MAX_MEMSTORE_TS_KEY)
540          || Bytes.equals(e.getKey(), HFileInfo.CREATE_TIME_TS)
541          || Bytes.equals(e.getKey(), HStoreFile.BULKLOAD_TIME_KEY)) {
542        out.println(Bytes.toLong(e.getValue()));
543      } else if (Bytes.equals(e.getKey(), HStoreFile.TIMERANGE_KEY)) {
544        TimeRangeTracker timeRangeTracker = TimeRangeTracker.parseFrom(e.getValue());
545        out.println(timeRangeTracker.getMin() + "...." + timeRangeTracker.getMax());
546      } else if (Bytes.equals(e.getKey(), HFileInfo.AVG_KEY_LEN)
547          || Bytes.equals(e.getKey(), HFileInfo.AVG_VALUE_LEN)
548          || Bytes.equals(e.getKey(), HFileWriterImpl.KEY_VALUE_VERSION)
549          || Bytes.equals(e.getKey(), HFileInfo.MAX_TAGS_LEN)) {
550        out.println(Bytes.toInt(e.getValue()));
551      } else if (Bytes.equals(e.getKey(), HStoreFile.MAJOR_COMPACTION_KEY)
552          || Bytes.equals(e.getKey(), HFileInfo.TAGS_COMPRESSED)
553          || Bytes.equals(e.getKey(), HStoreFile.EXCLUDE_FROM_MINOR_COMPACTION_KEY)) {
554        out.println(Bytes.toBoolean(e.getValue()));
555      } else if (Bytes.equals(e.getKey(), HFileInfo.LASTKEY)) {
556        out.println(new KeyValue.KeyOnlyKeyValue(e.getValue()).toString());
557      } else {
558        out.println(Bytes.toStringBinary(e.getValue()));
559      }
560    }
561
562    try {
563      out.println("Mid-key: " + reader.midKey().map(CellUtil::getCellKeyAsString));
564    } catch (Exception e) {
565      out.println ("Unable to retrieve the midkey");
566    }
567
568    // Printing general bloom information
569    DataInput bloomMeta = reader.getGeneralBloomFilterMetadata();
570    BloomFilter bloomFilter = null;
571    if (bloomMeta != null)
572      bloomFilter = BloomFilterFactory.createFromMeta(bloomMeta, reader);
573
574    out.println("Bloom filter:");
575    if (bloomFilter != null) {
576      out.println(FOUR_SPACES + bloomFilter.toString().replaceAll(
577          BloomFilterUtil.STATS_RECORD_SEP, "\n" + FOUR_SPACES));
578    } else {
579      out.println(FOUR_SPACES + "Not present");
580    }
581
582    // Printing delete bloom information
583    bloomMeta = reader.getDeleteBloomFilterMetadata();
584    bloomFilter = null;
585    if (bloomMeta != null)
586      bloomFilter = BloomFilterFactory.createFromMeta(bloomMeta, reader);
587
588    out.println("Delete Family Bloom filter:");
589    if (bloomFilter != null) {
590      out.println(FOUR_SPACES
591          + bloomFilter.toString().replaceAll(BloomFilterUtil.STATS_RECORD_SEP,
592              "\n" + FOUR_SPACES));
593    } else {
594      out.println(FOUR_SPACES + "Not present");
595    }
596  }
597
598  private static class KeyValueStatsCollector {
599    private final MetricRegistry metricsRegistry = new MetricRegistry();
600    private final ByteArrayOutputStream metricsOutput = new ByteArrayOutputStream();
601    private final SimpleReporter simpleReporter = SimpleReporter.forRegistry(metricsRegistry).
602        outputTo(new PrintStream(metricsOutput)).filter(MetricFilter.ALL).build();
603
604    Histogram keyLen = metricsRegistry.histogram(name(HFilePrettyPrinter.class, "Key length"));
605    Histogram valLen = metricsRegistry.histogram(name(HFilePrettyPrinter.class, "Val length"));
606    Histogram rowSizeBytes = metricsRegistry.histogram(
607      name(HFilePrettyPrinter.class, "Row size (bytes)"));
608    Histogram rowSizeCols = metricsRegistry.histogram(
609      name(HFilePrettyPrinter.class, "Row size (columns)"));
610
611    long curRowBytes = 0;
612    long curRowCols = 0;
613
614    byte[] biggestRow = null;
615
616    private Cell prevCell = null;
617    private long maxRowBytes = 0;
618    private long curRowKeyLength;
619
620    public void collect(Cell cell) {
621      valLen.update(cell.getValueLength());
622      if (prevCell != null &&
623          CellComparator.getInstance().compareRows(prevCell, cell) != 0) {
624        // new row
625        collectRow();
626      }
627      curRowBytes += cell.getSerializedSize();
628      curRowKeyLength = KeyValueUtil.keyLength(cell);
629      curRowCols++;
630      prevCell = cell;
631    }
632
633    private void collectRow() {
634      rowSizeBytes.update(curRowBytes);
635      rowSizeCols.update(curRowCols);
636      keyLen.update(curRowKeyLength);
637
638      if (curRowBytes > maxRowBytes && prevCell != null) {
639        biggestRow = CellUtil.cloneRow(prevCell);
640        maxRowBytes = curRowBytes;
641      }
642
643      curRowBytes = 0;
644      curRowCols = 0;
645    }
646
647    public void finish() {
648      if (curRowCols > 0) {
649        collectRow();
650      }
651    }
652
653    @Override
654    public String toString() {
655      if (prevCell == null)
656        return "no data available for statistics";
657
658      // Dump the metrics to the output stream
659      simpleReporter.stop();
660      simpleReporter.report();
661
662      return
663              metricsOutput.toString() +
664                      "Key of biggest row: " + Bytes.toStringBinary(biggestRow);
665    }
666  }
667
668  /**
669   * Almost identical to ConsoleReporter, but extending ScheduledReporter,
670   * as extending ConsoleReporter in this version of dropwizard is now too much trouble.
671   */
672  private static class SimpleReporter extends ScheduledReporter {
673    /**
674     * Returns a new {@link Builder} for {@link ConsoleReporter}.
675     *
676     * @param registry the registry to report
677     * @return a {@link Builder} instance for a {@link ConsoleReporter}
678     */
679    public static Builder forRegistry(MetricRegistry registry) {
680      return new Builder(registry);
681    }
682
683    /**
684     * A builder for {@link SimpleReporter} instances. Defaults to using the default locale and
685     * time zone, writing to {@code System.out}, converting rates to events/second, converting
686     * durations to milliseconds, and not filtering metrics.
687     */
688    public static class Builder {
689      private final MetricRegistry registry;
690      private PrintStream output;
691      private Locale locale;
692      private TimeZone timeZone;
693      private TimeUnit rateUnit;
694      private TimeUnit durationUnit;
695      private MetricFilter filter;
696
697      private Builder(MetricRegistry registry) {
698        this.registry = registry;
699        this.output = System.out;
700        this.locale = Locale.getDefault();
701        this.timeZone = TimeZone.getDefault();
702        this.rateUnit = TimeUnit.SECONDS;
703        this.durationUnit = TimeUnit.MILLISECONDS;
704        this.filter = MetricFilter.ALL;
705      }
706
707      /**
708       * Write to the given {@link PrintStream}.
709       *
710       * @param output a {@link PrintStream} instance.
711       * @return {@code this}
712       */
713      public Builder outputTo(PrintStream output) {
714        this.output = output;
715        return this;
716      }
717
718      /**
719       * Only report metrics which match the given filter.
720       *
721       * @param filter a {@link MetricFilter}
722       * @return {@code this}
723       */
724      public Builder filter(MetricFilter filter) {
725        this.filter = filter;
726        return this;
727      }
728
729      /**
730       * Builds a {@link ConsoleReporter} with the given properties.
731       *
732       * @return a {@link ConsoleReporter}
733       */
734      public SimpleReporter build() {
735        return new SimpleReporter(registry,
736            output,
737            locale,
738            timeZone,
739            rateUnit,
740            durationUnit,
741            filter);
742      }
743    }
744
745    private final PrintStream output;
746    private final Locale locale;
747    private final DateFormat dateFormat;
748
749    private SimpleReporter(MetricRegistry registry,
750                            PrintStream output,
751                            Locale locale,
752                            TimeZone timeZone,
753                            TimeUnit rateUnit,
754                            TimeUnit durationUnit,
755                            MetricFilter filter) {
756      super(registry, "simple-reporter", filter, rateUnit, durationUnit);
757      this.output = output;
758      this.locale = locale;
759
760      this.dateFormat = DateFormat.getDateTimeInstance(DateFormat.SHORT,
761          DateFormat.MEDIUM,
762          locale);
763      dateFormat.setTimeZone(timeZone);
764    }
765
766    @Override
767    public void report(SortedMap<String, Gauge> gauges,
768                       SortedMap<String, Counter> counters,
769                       SortedMap<String, Histogram> histograms,
770                       SortedMap<String, Meter> meters,
771                       SortedMap<String, Timer> timers) {
772      // we know we only have histograms
773      if (!histograms.isEmpty()) {
774        for (Map.Entry<String, Histogram> entry : histograms.entrySet()) {
775          output.print("   " + StringUtils.substringAfterLast(entry.getKey(), "."));
776          output.println(':');
777          printHistogram(entry.getValue());
778        }
779        output.println();
780      }
781
782      output.println();
783      output.flush();
784    }
785
786    private void printHistogram(Histogram histogram) {
787      Snapshot snapshot = histogram.getSnapshot();
788      output.printf(locale, "               min = %d%n", snapshot.getMin());
789      output.printf(locale, "               max = %d%n", snapshot.getMax());
790      output.printf(locale, "              mean = %2.2f%n", snapshot.getMean());
791      output.printf(locale, "            stddev = %2.2f%n", snapshot.getStdDev());
792      output.printf(locale, "            median = %2.2f%n", snapshot.getMedian());
793      output.printf(locale, "              75%% <= %2.2f%n", snapshot.get75thPercentile());
794      output.printf(locale, "              95%% <= %2.2f%n", snapshot.get95thPercentile());
795      output.printf(locale, "              98%% <= %2.2f%n", snapshot.get98thPercentile());
796      output.printf(locale, "              99%% <= %2.2f%n", snapshot.get99thPercentile());
797      output.printf(locale, "            99.9%% <= %2.2f%n", snapshot.get999thPercentile());
798      output.printf(locale, "             count = %d%n", histogram.getCount());
799    }
800  }
801
802  public static void main(String[] args) throws Exception {
803    Configuration conf = HBaseConfiguration.create();
804    // no need for a block cache
805    conf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0);
806    int ret = ToolRunner.run(conf, new HFilePrettyPrinter(), args);
807    System.exit(ret);
808  }
809}