001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.wal;
019
020import java.io.FileNotFoundException;
021import java.io.IOException;
022import java.io.PrintStream;
023import java.util.ArrayList;
024import java.util.Date;
025import java.util.HashMap;
026import java.util.Iterator;
027import java.util.List;
028import java.util.Map;
029
030import org.apache.hadoop.conf.Configuration;
031import org.apache.hadoop.fs.FileSystem;
032import org.apache.hadoop.fs.Path;
033import org.apache.hadoop.hbase.Cell;
034import org.apache.hadoop.hbase.CellUtil;
035import org.apache.hadoop.hbase.HBaseConfiguration;
036import org.apache.hadoop.hbase.HBaseInterfaceAudience;
037import org.apache.hadoop.hbase.PrivateCellUtil;
038import org.apache.hadoop.hbase.Tag;
039import org.apache.hadoop.hbase.regionserver.wal.ProtobufLogReader;
040import org.apache.hadoop.hbase.util.Bytes;
041import org.apache.hadoop.hbase.util.FSUtils;
042import org.apache.yetus.audience.InterfaceAudience;
043import org.apache.yetus.audience.InterfaceStability;
044
045import org.apache.hbase.thirdparty.org.apache.commons.cli.CommandLine;
046import org.apache.hbase.thirdparty.org.apache.commons.cli.CommandLineParser;
047import org.apache.hbase.thirdparty.org.apache.commons.cli.HelpFormatter;
048import org.apache.hbase.thirdparty.org.apache.commons.cli.Options;
049import org.apache.hbase.thirdparty.org.apache.commons.cli.ParseException;
050import org.apache.hbase.thirdparty.org.apache.commons.cli.PosixParser;
051
052import com.fasterxml.jackson.databind.ObjectMapper;
053
054/**
055 * WALPrettyPrinter prints the contents of a given WAL with a variety of
056 * options affecting formatting and extent of content.
057 *
058 * It targets two usage cases: pretty printing for ease of debugging directly by
059 * humans, and JSON output for consumption by monitoring and/or maintenance
060 * scripts.
061 *
062 * It can filter by row, region, or sequence id.
063 *
064 * It can also toggle output of values.
065 *
066 */
067@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.TOOLS)
068@InterfaceStability.Evolving
069public class WALPrettyPrinter {
070  private boolean outputValues;
071  private boolean outputJSON;
072  // The following enable filtering by sequence, region, and row, respectively
073  private long sequence;
074  private String region;
075  private String row;
076  // enable in order to output a single list of transactions from several files
077  private boolean persistentOutput;
078  private boolean firstTxn;
079  // useful for programmatic capture of JSON output
080  private PrintStream out;
081  // for JSON encoding
082  private static final ObjectMapper MAPPER = new ObjectMapper();
083  //allows for jumping straight to a given portion of the file
084  private long position;
085
086  /**
087   * Basic constructor that simply initializes values to reasonable defaults.
088   */
089  public WALPrettyPrinter() {
090    outputValues = false;
091    outputJSON = false;
092    sequence = -1;
093    region = null;
094    row = null;
095    persistentOutput = false;
096    firstTxn = true;
097    out = System.out;
098  }
099
100  /**
101   * Fully specified constructor.
102   *
103   * @param outputValues
104   *          when true, enables output of values along with other log
105   *          information
106   * @param outputJSON
107   *          when true, enables output in JSON format rather than a
108   *          "pretty string"
109   * @param sequence
110   *          when nonnegative, serves as a filter; only log entries with this
111   *          sequence id will be printed
112   * @param region
113   *          when not null, serves as a filter; only log entries from this
114   *          region will be printed
115   * @param row
116   *          when not null, serves as a filter; only log entries from this row
117   *          will be printed
118   * @param persistentOutput
119   *          keeps a single list running for multiple files. if enabled, the
120   *          endPersistentOutput() method must be used!
121   * @param out
122   *          Specifies an alternative to stdout for the destination of this
123   *          PrettyPrinter's output.
124   */
125  public WALPrettyPrinter(boolean outputValues, boolean outputJSON,
126      long sequence, String region, String row, boolean persistentOutput,
127      PrintStream out) {
128    this.outputValues = outputValues;
129    this.outputJSON = outputJSON;
130    this.sequence = sequence;
131    this.region = region;
132    this.row = row;
133    this.persistentOutput = persistentOutput;
134    if (persistentOutput) {
135      beginPersistentOutput();
136    }
137    this.out = out;
138    this.firstTxn = true;
139  }
140
141  /**
142   * turns value output on
143   */
144  public void enableValues() {
145    outputValues = true;
146  }
147
148  /**
149   * turns value output off
150   */
151  public void disableValues() {
152    outputValues = false;
153  }
154
155  /**
156   * turns JSON output on
157   */
158  public void enableJSON() {
159    outputJSON = true;
160  }
161
162  /**
163   * turns JSON output off, and turns on "pretty strings" for human consumption
164   */
165  public void disableJSON() {
166    outputJSON = false;
167  }
168
169  /**
170   * sets the region by which output will be filtered
171   *
172   * @param sequence
173   *          when nonnegative, serves as a filter; only log entries with this
174   *          sequence id will be printed
175   */
176  public void setSequenceFilter(long sequence) {
177    this.sequence = sequence;
178  }
179
180  /**
181   * sets the region by which output will be filtered
182   *
183   * @param region
184   *          when not null, serves as a filter; only log entries from this
185   *          region will be printed
186   */
187  public void setRegionFilter(String region) {
188    this.region = region;
189  }
190
191  /**
192   * sets the region by which output will be filtered
193   *
194   * @param row
195   *          when not null, serves as a filter; only log entries from this row
196   *          will be printed
197   */
198  public void setRowFilter(String row) {
199    this.row = row;
200  }
201
202  /**
203   * sets the position to start seeking the WAL file
204   * @param position
205   *          initial position to start seeking the given WAL file
206   */
207  public void setPosition(long position) {
208    this.position = position;
209  }
210
211  /**
212   * enables output as a single, persistent list. at present, only relevant in
213   * the case of JSON output.
214   */
215  public void beginPersistentOutput() {
216    if (persistentOutput)
217      return;
218    persistentOutput = true;
219    firstTxn = true;
220    if (outputJSON)
221      out.print("[");
222  }
223
224  /**
225   * ends output of a single, persistent list. at present, only relevant in the
226   * case of JSON output.
227   */
228  public void endPersistentOutput() {
229    if (!persistentOutput)
230      return;
231    persistentOutput = false;
232    if (outputJSON)
233      out.print("]");
234  }
235
236  /**
237   * reads a log file and outputs its contents, one transaction at a time, as
238   * specified by the currently configured options
239   *
240   * @param conf
241   *          the HBase configuration relevant to this log file
242   * @param p
243   *          the path of the log file to be read
244   * @throws IOException
245   *           may be unable to access the configured filesystem or requested
246   *           file.
247   */
248  public void processFile(final Configuration conf, final Path p)
249      throws IOException {
250    FileSystem fs = p.getFileSystem(conf);
251    if (!fs.exists(p)) {
252      throw new FileNotFoundException(p.toString());
253    }
254    if (!fs.isFile(p)) {
255      throw new IOException(p + " is not a file");
256    }
257
258    WAL.Reader log = WALFactory.createReader(fs, p, conf);
259
260    if (log instanceof ProtobufLogReader) {
261      List<String> writerClsNames = ((ProtobufLogReader) log).getWriterClsNames();
262      if (writerClsNames != null && writerClsNames.size() > 0) {
263        out.print("Writer Classes: ");
264        for (int i = 0; i < writerClsNames.size(); i++) {
265          out.print(writerClsNames.get(i));
266          if (i != writerClsNames.size() - 1) {
267            out.print(" ");
268          }
269        }
270        out.println();
271      }
272
273      String cellCodecClsName = ((ProtobufLogReader) log).getCodecClsName();
274      if (cellCodecClsName != null) {
275        out.println("Cell Codec Class: " + cellCodecClsName);
276      }
277    }
278
279    if (outputJSON && !persistentOutput) {
280      out.print("[");
281      firstTxn = true;
282    }
283
284    if (position > 0) {
285      log.seek(position);
286    }
287
288    try {
289      WAL.Entry entry;
290      while ((entry = log.next()) != null) {
291        WALKey key = entry.getKey();
292        WALEdit edit = entry.getEdit();
293        // begin building a transaction structure
294        Map<String, Object> txn = key.toStringMap();
295        long writeTime = key.getWriteTime();
296        // check output filters
297        if (sequence >= 0 && ((Long) txn.get("sequence")) != sequence)
298          continue;
299        if (region != null && !((String) txn.get("region")).equals(region))
300          continue;
301        // initialize list into which we will store atomic actions
302        List<Map> actions = new ArrayList<>();
303        for (Cell cell : edit.getCells()) {
304          // add atomic operation to txn
305          Map<String, Object> op = new HashMap<>(toStringMap(cell));
306          if (outputValues) op.put("value", Bytes.toStringBinary(CellUtil.cloneValue(cell)));
307          // check row output filter
308          if (row == null || ((String) op.get("row")).equals(row)) {
309            actions.add(op);
310          }
311          op.put("total_size_sum", PrivateCellUtil.estimatedSizeOfCell(cell));
312        }
313        if (actions.isEmpty())
314          continue;
315        txn.put("actions", actions);
316        if (outputJSON) {
317          // JSON output is a straightforward "toString" on the txn object
318          if (firstTxn)
319            firstTxn = false;
320          else
321            out.print(",");
322          // encode and print JSON
323          out.print(MAPPER.writeValueAsString(txn));
324        } else {
325          // Pretty output, complete with indentation by atomic action
326          out.println("Sequence=" + txn.get("sequence") + " "
327              + ", region=" + txn.get("region") + " at write timestamp=" + new Date(writeTime));
328          for (int i = 0; i < actions.size(); i++) {
329            Map op = actions.get(i);
330            out.println("row=" + op.get("row") +
331                ", column=" + op.get("family") + ":" + op.get("qualifier"));
332            if (op.get("tag") != null) {
333              out.println("    tag: " + op.get("tag"));
334            }
335            if (outputValues) out.println("    value: " + op.get("value"));
336            out.println("cell total size sum: " + op.get("total_size_sum"));
337          }
338        }
339        out.println("edit heap size: " + entry.getEdit().heapSize());
340        out.println("position: " + log.getPosition());
341      }
342    } finally {
343      log.close();
344    }
345    if (outputJSON && !persistentOutput) {
346      out.print("]");
347    }
348  }
349
350  private static Map<String, Object> toStringMap(Cell cell) {
351    Map<String, Object> stringMap = new HashMap<>();
352    stringMap.put("row",
353        Bytes.toStringBinary(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength()));
354    stringMap.put("family", Bytes.toStringBinary(cell.getFamilyArray(), cell.getFamilyOffset(),
355                cell.getFamilyLength()));
356    stringMap.put("qualifier",
357        Bytes.toStringBinary(cell.getQualifierArray(), cell.getQualifierOffset(),
358            cell.getQualifierLength()));
359    stringMap.put("timestamp", cell.getTimestamp());
360    stringMap.put("vlen", cell.getValueLength());
361    if (cell.getTagsLength() > 0) {
362      List<String> tagsString = new ArrayList<>();
363      Iterator<Tag> tagsIterator = PrivateCellUtil.tagsIterator(cell);
364      while (tagsIterator.hasNext()) {
365        Tag tag = tagsIterator.next();
366        tagsString
367            .add((tag.getType()) + ":" + Bytes.toStringBinary(Tag.cloneValue(tag)));
368      }
369      stringMap.put("tag", tagsString);
370    }
371    return stringMap;
372  }
373
374  public static void main(String[] args) throws IOException {
375    run(args);
376  }
377
378  /**
379   * Pass one or more log file names and formatting options and it will dump out
380   * a text version of the contents on <code>stdout</code>.
381   *
382   * @param args
383   *          Command line arguments
384   * @throws IOException
385   *           Thrown upon file system errors etc.
386   */
387  public static void run(String[] args) throws IOException {
388    // create options
389    Options options = new Options();
390    options.addOption("h", "help", false, "Output help message");
391    options.addOption("j", "json", false, "Output JSON");
392    options.addOption("p", "printvals", false, "Print values");
393    options.addOption("r", "region", true,
394        "Region to filter by. Pass encoded region name; e.g. '9192caead6a5a20acb4454ffbc79fa14'");
395    options.addOption("s", "sequence", true,
396        "Sequence to filter by. Pass sequence number.");
397    options.addOption("w", "row", true, "Row to filter by. Pass row name.");
398    options.addOption("g", "goto", true, "Position to seek to in the file");
399
400    WALPrettyPrinter printer = new WALPrettyPrinter();
401    CommandLineParser parser = new PosixParser();
402    List<?> files = null;
403    try {
404      CommandLine cmd = parser.parse(options, args);
405      files = cmd.getArgList();
406      if (files.isEmpty() || cmd.hasOption("h")) {
407        HelpFormatter formatter = new HelpFormatter();
408        formatter.printHelp("WAL <filename...>", options, true);
409        System.exit(-1);
410      }
411      // configure the pretty printer using command line options
412      if (cmd.hasOption("p"))
413        printer.enableValues();
414      if (cmd.hasOption("j"))
415        printer.enableJSON();
416      if (cmd.hasOption("r"))
417        printer.setRegionFilter(cmd.getOptionValue("r"));
418      if (cmd.hasOption("s"))
419        printer.setSequenceFilter(Long.parseLong(cmd.getOptionValue("s")));
420      if (cmd.hasOption("w"))
421        printer.setRowFilter(cmd.getOptionValue("w"));
422      if (cmd.hasOption("g")) {
423        printer.setPosition(Long.parseLong(cmd.getOptionValue("g")));
424      }
425    } catch (ParseException e) {
426      e.printStackTrace();
427      HelpFormatter formatter = new HelpFormatter();
428      formatter.printHelp("HFile filename(s) ", options, true);
429      System.exit(-1);
430    }
431    // get configuration, file system, and process the given files
432    Configuration conf = HBaseConfiguration.create();
433    FSUtils.setFsDefault(conf, FSUtils.getRootDir(conf));
434
435    // begin output
436    printer.beginPersistentOutput();
437    for (Object f : files) {
438      Path file = new Path((String) f);
439      FileSystem fs = file.getFileSystem(conf);
440      if (!fs.exists(file)) {
441        System.err.println("ERROR, file doesnt exist: " + file);
442        return;
443      }
444      printer.processFile(conf, file);
445    }
446    printer.endPersistentOutput();
447  }
448}