001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.wal;
019
020import java.io.FileNotFoundException;
021import java.io.IOException;
022import java.io.PrintStream;
023import java.util.ArrayList;
024import java.util.Date;
025import java.util.HashMap;
026import java.util.Iterator;
027import java.util.List;
028import java.util.Map;
029import org.apache.hadoop.conf.Configuration;
030import org.apache.hadoop.fs.FileSystem;
031import org.apache.hadoop.fs.Path;
032import org.apache.hadoop.hbase.Cell;
033import org.apache.hadoop.hbase.CellUtil;
034import org.apache.hadoop.hbase.HBaseConfiguration;
035import org.apache.hadoop.hbase.HBaseInterfaceAudience;
036import org.apache.hadoop.hbase.PrivateCellUtil;
037import org.apache.hadoop.hbase.TableName;
038import org.apache.hadoop.hbase.Tag;
039import org.apache.hadoop.hbase.regionserver.wal.ProtobufLogReader;
040import org.apache.hadoop.hbase.util.Bytes;
041import org.apache.hadoop.hbase.util.FSUtils;
042import org.apache.hadoop.hbase.util.GsonUtil;
043import org.apache.yetus.audience.InterfaceAudience;
044import org.apache.yetus.audience.InterfaceStability;
045import org.slf4j.Logger;
046import org.slf4j.LoggerFactory;
047import org.apache.hbase.thirdparty.com.google.gson.Gson;
048import org.apache.hbase.thirdparty.org.apache.commons.cli.CommandLine;
049import org.apache.hbase.thirdparty.org.apache.commons.cli.CommandLineParser;
050import org.apache.hbase.thirdparty.org.apache.commons.cli.HelpFormatter;
051import org.apache.hbase.thirdparty.org.apache.commons.cli.Options;
052import org.apache.hbase.thirdparty.org.apache.commons.cli.ParseException;
053import org.apache.hbase.thirdparty.org.apache.commons.cli.PosixParser;
054
055/**
056 * WALPrettyPrinter prints the contents of a given WAL with a variety of
057 * options affecting formatting and extent of content.
058 *
059 * It targets two usage cases: pretty printing for ease of debugging directly by
060 * humans, and JSON output for consumption by monitoring and/or maintenance
061 * scripts.
062 *
063 * It can filter by row, region, or sequence id.
064 *
065 * It can also toggle output of values.
066 *
067 */
068@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.TOOLS)
069@InterfaceStability.Evolving
070public class WALPrettyPrinter {
071  private static final Logger LOG = LoggerFactory.getLogger(WALPrettyPrinter.class);
072
073  // Output template for pretty printing.
074  private static final String outputTmpl =
075      "Sequence=%s, table=%s, region=%s, at write timestamp=%s";
076
077  private boolean outputValues;
078  private boolean outputJSON;
079  // The following enable filtering by sequence, region, and row, respectively
080  private long sequence;
081  private String table;
082  private String region;
083  private String row;
084  // enable in order to output a single list of transactions from several files
085  private boolean persistentOutput;
086  private boolean firstTxn;
087  // useful for programmatic capture of JSON output
088  private PrintStream out;
089  // for JSON encoding
090  private static final Gson GSON = GsonUtil.createGson().create();
091  //allows for jumping straight to a given portion of the file
092  private long position;
093
094  /**
095   * Basic constructor that simply initializes values to reasonable defaults.
096   */
097  public WALPrettyPrinter() {
098    outputValues = false;
099    outputJSON = false;
100    sequence = -1;
101    table = null;
102    region = null;
103    row = null;
104    persistentOutput = false;
105    firstTxn = true;
106    out = System.out;
107  }
108
109  /**
110   * Fully specified constructor.
111   *
112   * @param outputValues
113   *          when true, enables output of values along with other log
114   *          information
115   * @param outputJSON
116   *          when true, enables output in JSON format rather than a
117   *          "pretty string"
118   * @param sequence
119   *          when nonnegative, serves as a filter; only log entries with this
120   *          sequence id will be printed
121   * @param table
122   *          when non null, serves as a filter. only entries corresponding to this
123   *          table will be printed.
124   * @param region
125   *          when not null, serves as a filter; only log entries from this
126   *          region will be printed
127   * @param row
128   *          when not null, serves as a filter; only log entries from this row
129   *          will be printed
130   * @param persistentOutput
131   *          keeps a single list running for multiple files. if enabled, the
132   *          endPersistentOutput() method must be used!
133   * @param out
134   *          Specifies an alternative to stdout for the destination of this
135   *          PrettyPrinter's output.
136   */
137  public WALPrettyPrinter(boolean outputValues, boolean outputJSON,
138      long sequence, String table, String region, String row, boolean persistentOutput,
139      PrintStream out) {
140    this.outputValues = outputValues;
141    this.outputJSON = outputJSON;
142    this.sequence = sequence;
143    this.table = table;
144    this.region = region;
145    this.row = row;
146    this.persistentOutput = persistentOutput;
147    if (persistentOutput) {
148      beginPersistentOutput();
149    }
150    this.out = out;
151    this.firstTxn = true;
152  }
153
154  /**
155   * turns value output on
156   */
157  public void enableValues() {
158    outputValues = true;
159  }
160
161  /**
162   * turns value output off
163   */
164  public void disableValues() {
165    outputValues = false;
166  }
167
168  /**
169   * turns JSON output on
170   */
171  public void enableJSON() {
172    outputJSON = true;
173  }
174
175  /**
176   * turns JSON output off, and turns on "pretty strings" for human consumption
177   */
178  public void disableJSON() {
179    outputJSON = false;
180  }
181
182  /**
183   * sets the region by which output will be filtered
184   *
185   * @param sequence
186   *          when nonnegative, serves as a filter; only log entries with this
187   *          sequence id will be printed
188   */
189  public void setSequenceFilter(long sequence) {
190    this.sequence = sequence;
191  }
192
193  /**
194   * Sets the table filter. Only log entries for this table are printed.
195   * @param table table name to set.
196   */
197  public void setTableFilter(String table) {
198    this.table = table;
199  }
200  /**
201   * sets the region by which output will be filtered
202   *
203   * @param region
204   *          when not null, serves as a filter; only log entries from this
205   *          region will be printed
206   */
207  public void setRegionFilter(String region) {
208    this.region = region;
209  }
210
211  /**
212   * sets the region by which output will be filtered
213   *
214   * @param row
215   *          when not null, serves as a filter; only log entries from this row
216   *          will be printed
217   */
218  public void setRowFilter(String row) {
219    this.row = row;
220  }
221
222  /**
223   * sets the position to start seeking the WAL file
224   * @param position
225   *          initial position to start seeking the given WAL file
226   */
227  public void setPosition(long position) {
228    this.position = position;
229  }
230
231  /**
232   * enables output as a single, persistent list. at present, only relevant in
233   * the case of JSON output.
234   */
235  public void beginPersistentOutput() {
236    if (persistentOutput) {
237      return;
238    }
239    persistentOutput = true;
240    firstTxn = true;
241    if (outputJSON) {
242      out.print("[");
243    }
244  }
245
246  /**
247   * ends output of a single, persistent list. at present, only relevant in the
248   * case of JSON output.
249   */
250  public void endPersistentOutput() {
251    if (!persistentOutput) {
252      return;
253    }
254    persistentOutput = false;
255    if (outputJSON) {
256      out.print("]");
257    }
258  }
259
260  /**
261   * reads a log file and outputs its contents, one transaction at a time, as
262   * specified by the currently configured options
263   *
264   * @param conf
265   *          the HBase configuration relevant to this log file
266   * @param p
267   *          the path of the log file to be read
268   * @throws IOException
269   *           may be unable to access the configured filesystem or requested
270   *           file.
271   */
272  public void processFile(final Configuration conf, final Path p)
273      throws IOException {
274    FileSystem fs = p.getFileSystem(conf);
275    if (!fs.exists(p)) {
276      throw new FileNotFoundException(p.toString());
277    }
278    if (!fs.isFile(p)) {
279      throw new IOException(p + " is not a file");
280    }
281
282    WAL.Reader log = WALFactory.createReader(fs, p, conf);
283
284    if (log instanceof ProtobufLogReader) {
285      List<String> writerClsNames = ((ProtobufLogReader) log).getWriterClsNames();
286      if (writerClsNames != null && writerClsNames.size() > 0) {
287        out.print("Writer Classes: ");
288        for (int i = 0; i < writerClsNames.size(); i++) {
289          out.print(writerClsNames.get(i));
290          if (i != writerClsNames.size() - 1) {
291            out.print(" ");
292          }
293        }
294        out.println();
295      }
296
297      String cellCodecClsName = ((ProtobufLogReader) log).getCodecClsName();
298      if (cellCodecClsName != null) {
299        out.println("Cell Codec Class: " + cellCodecClsName);
300      }
301    }
302
303    if (outputJSON && !persistentOutput) {
304      out.print("[");
305      firstTxn = true;
306    }
307
308    if (position > 0) {
309      log.seek(position);
310    }
311
312    try {
313      WAL.Entry entry;
314      while ((entry = log.next()) != null) {
315        WALKey key = entry.getKey();
316        WALEdit edit = entry.getEdit();
317        // begin building a transaction structure
318        Map<String, Object> txn = key.toStringMap();
319        long writeTime = key.getWriteTime();
320        // check output filters
321        if (table != null && !((TableName) txn.get("table")).toString().equals(table)) {
322          continue;
323        }
324        if (sequence >= 0 && ((Long) txn.get("sequence")) != sequence) {
325          continue;
326        }
327        if (region != null && !txn.get("region").equals(region)) {
328          continue;
329        }
330        // initialize list into which we will store atomic actions
331        List<Map> actions = new ArrayList<>();
332        for (Cell cell : edit.getCells()) {
333          // add atomic operation to txn
334          Map<String, Object> op = new HashMap<>(toStringMap(cell));
335          if (outputValues) {
336            op.put("value", Bytes.toStringBinary(CellUtil.cloneValue(cell)));
337          }
338          // check row output filter
339          if (row == null || ((String) op.get("row")).equals(row)) {
340            actions.add(op);
341          }
342          op.put("total_size_sum", cell.heapSize());
343        }
344        if (actions.isEmpty()) {
345          continue;
346        }
347        txn.put("actions", actions);
348        if (outputJSON) {
349          // JSON output is a straightforward "toString" on the txn object
350          if (firstTxn) {
351            firstTxn = false;
352          } else {
353            out.print(",");
354          }
355          // encode and print JSON
356          out.print(GSON.toJson(txn));
357        } else {
358          // Pretty output, complete with indentation by atomic action
359          out.println(String.format(outputTmpl,
360              txn.get("sequence"), txn.get("table"), txn.get("region"), new Date(writeTime)));
361          for (int i = 0; i < actions.size(); i++) {
362            Map op = actions.get(i);
363            out.println("row=" + op.get("row") +
364                ", column=" + op.get("family") + ":" + op.get("qualifier"));
365            if (op.get("tag") != null) {
366              out.println("    tag: " + op.get("tag"));
367            }
368            if (outputValues) {
369              out.println("    value: " + op.get("value"));
370            }
371            out.println("cell total size sum: " + op.get("total_size_sum"));
372          }
373        }
374        out.println("edit heap size: " + entry.getEdit().heapSize());
375        out.println("position: " + log.getPosition());
376      }
377    } finally {
378      log.close();
379    }
380    if (outputJSON && !persistentOutput) {
381      out.print("]");
382    }
383  }
384
385  private static Map<String, Object> toStringMap(Cell cell) {
386    Map<String, Object> stringMap = new HashMap<>();
387    stringMap.put("row",
388        Bytes.toStringBinary(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength()));
389    stringMap.put("family", Bytes.toStringBinary(cell.getFamilyArray(), cell.getFamilyOffset(),
390                cell.getFamilyLength()));
391    stringMap.put("qualifier",
392        Bytes.toStringBinary(cell.getQualifierArray(), cell.getQualifierOffset(),
393            cell.getQualifierLength()));
394    stringMap.put("timestamp", cell.getTimestamp());
395    stringMap.put("vlen", cell.getValueLength());
396    if (cell.getTagsLength() > 0) {
397      List<String> tagsString = new ArrayList<>();
398      Iterator<Tag> tagsIterator = PrivateCellUtil.tagsIterator(cell);
399      while (tagsIterator.hasNext()) {
400        Tag tag = tagsIterator.next();
401        tagsString
402            .add((tag.getType()) + ":" + Bytes.toStringBinary(Tag.cloneValue(tag)));
403      }
404      stringMap.put("tag", tagsString);
405    }
406    return stringMap;
407  }
408
409  public static void main(String[] args) throws IOException {
410    run(args);
411  }
412
413  /**
414   * Pass one or more log file names and formatting options and it will dump out
415   * a text version of the contents on <code>stdout</code>.
416   *
417   * @param args
418   *          Command line arguments
419   * @throws IOException
420   *           Thrown upon file system errors etc.
421   */
422  public static void run(String[] args) throws IOException {
423    // create options
424    Options options = new Options();
425    options.addOption("h", "help", false, "Output help message");
426    options.addOption("j", "json", false, "Output JSON");
427    options.addOption("p", "printvals", false, "Print values");
428    options.addOption("t", "table", true, "Table name to filter by.");
429    options.addOption("r", "region", true,
430        "Region to filter by. Pass encoded region name; e.g. '9192caead6a5a20acb4454ffbc79fa14'");
431    options.addOption("s", "sequence", true,
432        "Sequence to filter by. Pass sequence number.");
433    options.addOption("w", "row", true, "Row to filter by. Pass row name.");
434    options.addOption("g", "goto", true, "Position to seek to in the file");
435
436    WALPrettyPrinter printer = new WALPrettyPrinter();
437    CommandLineParser parser = new PosixParser();
438    List<?> files = null;
439    try {
440      CommandLine cmd = parser.parse(options, args);
441      files = cmd.getArgList();
442      if (files.isEmpty() || cmd.hasOption("h")) {
443        HelpFormatter formatter = new HelpFormatter();
444        formatter.printHelp("WAL <filename...>", options, true);
445        System.exit(-1);
446      }
447      // configure the pretty printer using command line options
448      if (cmd.hasOption("p")) {
449        printer.enableValues();
450      }
451      if (cmd.hasOption("j")) {
452        printer.enableJSON();
453      }
454      if (cmd.hasOption("t")) {
455        printer.setTableFilter(cmd.getOptionValue("t"));
456      }
457      if (cmd.hasOption("r")) {
458        printer.setRegionFilter(cmd.getOptionValue("r"));
459      }
460      if (cmd.hasOption("s")) {
461        printer.setSequenceFilter(Long.parseLong(cmd.getOptionValue("s")));
462      }
463      if (cmd.hasOption("w")) {
464        printer.setRowFilter(cmd.getOptionValue("w"));
465      }
466      if (cmd.hasOption("g")) {
467        printer.setPosition(Long.parseLong(cmd.getOptionValue("g")));
468      }
469    } catch (ParseException e) {
470      LOG.error("Failed to parse commandLine arguments", e);
471      HelpFormatter formatter = new HelpFormatter();
472      formatter.printHelp("HFile filename(s) ", options, true);
473      System.exit(-1);
474    }
475    // get configuration, file system, and process the given files
476    Configuration conf = HBaseConfiguration.create();
477    FSUtils.setFsDefault(conf, FSUtils.getRootDir(conf));
478
479    // begin output
480    printer.beginPersistentOutput();
481    for (Object f : files) {
482      Path file = new Path((String) f);
483      FileSystem fs = file.getFileSystem(conf);
484      if (!fs.exists(file)) {
485        System.err.println("ERROR, file doesnt exist: " + file);
486        return;
487      }
488      printer.processFile(conf, file);
489    }
490    printer.endPersistentOutput();
491  }
492}