001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.wal;
019
020import java.io.FileNotFoundException;
021import java.io.IOException;
022import java.io.PrintStream;
023import java.util.ArrayList;
024import java.util.Date;
025import java.util.HashMap;
026import java.util.Iterator;
027import java.util.List;
028import java.util.Map;
029
030import org.apache.hadoop.conf.Configuration;
031import org.apache.hadoop.fs.FileSystem;
032import org.apache.hadoop.fs.Path;
033import org.apache.hadoop.hbase.Cell;
034import org.apache.hadoop.hbase.CellUtil;
035import org.apache.hadoop.hbase.HBaseConfiguration;
036import org.apache.hadoop.hbase.HBaseInterfaceAudience;
037import org.apache.hadoop.hbase.PrivateCellUtil;
038import org.apache.hadoop.hbase.Tag;
039import org.apache.hadoop.hbase.regionserver.wal.ProtobufLogReader;
040import org.apache.hadoop.hbase.util.Bytes;
041import org.apache.hadoop.hbase.util.FSUtils;
042import org.apache.yetus.audience.InterfaceAudience;
043import org.apache.yetus.audience.InterfaceStability;
044
045import org.apache.hbase.thirdparty.org.apache.commons.cli.CommandLine;
046import org.apache.hbase.thirdparty.org.apache.commons.cli.CommandLineParser;
047import org.apache.hbase.thirdparty.org.apache.commons.cli.HelpFormatter;
048import org.apache.hbase.thirdparty.org.apache.commons.cli.Options;
049import org.apache.hbase.thirdparty.org.apache.commons.cli.ParseException;
050import org.apache.hbase.thirdparty.org.apache.commons.cli.PosixParser;
051
052import com.fasterxml.jackson.databind.ObjectMapper;
053
054/**
055 * WALPrettyPrinter prints the contents of a given WAL with a variety of
056 * options affecting formatting and extent of content.
057 *
058 * It targets two usage cases: pretty printing for ease of debugging directly by
059 * humans, and JSON output for consumption by monitoring and/or maintenance
060 * scripts.
061 *
062 * It can filter by row, region, or sequence id.
063 *
064 * It can also toggle output of values.
065 *
066 */
067@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.TOOLS)
068@InterfaceStability.Evolving
069public class WALPrettyPrinter {
070  private boolean outputValues;
071  private boolean outputJSON;
072  // The following enable filtering by sequence, region, and row, respectively
073  private long sequence;
074  private String region;
075  private String row;
076  // enable in order to output a single list of transactions from several files
077  private boolean persistentOutput;
078  private boolean firstTxn;
079  // useful for programmatic capture of JSON output
080  private PrintStream out;
081  // for JSON encoding
082  private static final ObjectMapper MAPPER = new ObjectMapper();
083
084  /**
085   * Basic constructor that simply initializes values to reasonable defaults.
086   */
087  public WALPrettyPrinter() {
088    outputValues = false;
089    outputJSON = false;
090    sequence = -1;
091    region = null;
092    row = null;
093    persistentOutput = false;
094    firstTxn = true;
095    out = System.out;
096  }
097
098  /**
099   * Fully specified constructor.
100   *
101   * @param outputValues
102   *          when true, enables output of values along with other log
103   *          information
104   * @param outputJSON
105   *          when true, enables output in JSON format rather than a
106   *          "pretty string"
107   * @param sequence
108   *          when nonnegative, serves as a filter; only log entries with this
109   *          sequence id will be printed
110   * @param region
111   *          when not null, serves as a filter; only log entries from this
112   *          region will be printed
113   * @param row
114   *          when not null, serves as a filter; only log entries from this row
115   *          will be printed
116   * @param persistentOutput
117   *          keeps a single list running for multiple files. if enabled, the
118   *          endPersistentOutput() method must be used!
119   * @param out
120   *          Specifies an alternative to stdout for the destination of this
121   *          PrettyPrinter's output.
122   */
123  public WALPrettyPrinter(boolean outputValues, boolean outputJSON,
124      long sequence, String region, String row, boolean persistentOutput,
125      PrintStream out) {
126    this.outputValues = outputValues;
127    this.outputJSON = outputJSON;
128    this.sequence = sequence;
129    this.region = region;
130    this.row = row;
131    this.persistentOutput = persistentOutput;
132    if (persistentOutput) {
133      beginPersistentOutput();
134    }
135    this.out = out;
136    this.firstTxn = true;
137  }
138
139  /**
140   * turns value output on
141   */
142  public void enableValues() {
143    outputValues = true;
144  }
145
146  /**
147   * turns value output off
148   */
149  public void disableValues() {
150    outputValues = false;
151  }
152
153  /**
154   * turns JSON output on
155   */
156  public void enableJSON() {
157    outputJSON = true;
158  }
159
160  /**
161   * turns JSON output off, and turns on "pretty strings" for human consumption
162   */
163  public void disableJSON() {
164    outputJSON = false;
165  }
166
167  /**
168   * sets the region by which output will be filtered
169   *
170   * @param sequence
171   *          when nonnegative, serves as a filter; only log entries with this
172   *          sequence id will be printed
173   */
174  public void setSequenceFilter(long sequence) {
175    this.sequence = sequence;
176  }
177
178  /**
179   * sets the region by which output will be filtered
180   *
181   * @param region
182   *          when not null, serves as a filter; only log entries from this
183   *          region will be printed
184   */
185  public void setRegionFilter(String region) {
186    this.region = region;
187  }
188
189  /**
190   * sets the region by which output will be filtered
191   *
192   * @param row
193   *          when not null, serves as a filter; only log entries from this row
194   *          will be printed
195   */
196  public void setRowFilter(String row) {
197    this.row = row;
198  }
199
200  /**
201   * enables output as a single, persistent list. at present, only relevant in
202   * the case of JSON output.
203   */
204  public void beginPersistentOutput() {
205    if (persistentOutput)
206      return;
207    persistentOutput = true;
208    firstTxn = true;
209    if (outputJSON)
210      out.print("[");
211  }
212
213  /**
214   * ends output of a single, persistent list. at present, only relevant in the
215   * case of JSON output.
216   */
217  public void endPersistentOutput() {
218    if (!persistentOutput)
219      return;
220    persistentOutput = false;
221    if (outputJSON)
222      out.print("]");
223  }
224
225  /**
226   * reads a log file and outputs its contents, one transaction at a time, as
227   * specified by the currently configured options
228   *
229   * @param conf
230   *          the HBase configuration relevant to this log file
231   * @param p
232   *          the path of the log file to be read
233   * @throws IOException
234   *           may be unable to access the configured filesystem or requested
235   *           file.
236   */
237  public void processFile(final Configuration conf, final Path p)
238      throws IOException {
239    FileSystem fs = p.getFileSystem(conf);
240    if (!fs.exists(p)) {
241      throw new FileNotFoundException(p.toString());
242    }
243    if (!fs.isFile(p)) {
244      throw new IOException(p + " is not a file");
245    }
246
247    WAL.Reader log = WALFactory.createReader(fs, p, conf);
248
249    if (log instanceof ProtobufLogReader) {
250      List<String> writerClsNames = ((ProtobufLogReader) log).getWriterClsNames();
251      if (writerClsNames != null && writerClsNames.size() > 0) {
252        out.print("Writer Classes: ");
253        for (int i = 0; i < writerClsNames.size(); i++) {
254          out.print(writerClsNames.get(i));
255          if (i != writerClsNames.size() - 1) {
256            out.print(" ");
257          }
258        }
259        out.println();
260      }
261
262      String cellCodecClsName = ((ProtobufLogReader) log).getCodecClsName();
263      if (cellCodecClsName != null) {
264        out.println("Cell Codec Class: " + cellCodecClsName);
265      }
266    }
267
268    if (outputJSON && !persistentOutput) {
269      out.print("[");
270      firstTxn = true;
271    }
272
273    try {
274      WAL.Entry entry;
275      while ((entry = log.next()) != null) {
276        WALKey key = entry.getKey();
277        WALEdit edit = entry.getEdit();
278        // begin building a transaction structure
279        Map<String, Object> txn = key.toStringMap();
280        long writeTime = key.getWriteTime();
281        // check output filters
282        if (sequence >= 0 && ((Long) txn.get("sequence")) != sequence)
283          continue;
284        if (region != null && !((String) txn.get("region")).equals(region))
285          continue;
286        // initialize list into which we will store atomic actions
287        List<Map> actions = new ArrayList<>();
288        for (Cell cell : edit.getCells()) {
289          // add atomic operation to txn
290          Map<String, Object> op = new HashMap<>(toStringMap(cell));
291          if (outputValues) op.put("value", Bytes.toStringBinary(CellUtil.cloneValue(cell)));
292          // check row output filter
293          if (row == null || ((String) op.get("row")).equals(row)) {
294            actions.add(op);
295          }
296        }
297        if (actions.isEmpty())
298          continue;
299        txn.put("actions", actions);
300        if (outputJSON) {
301          // JSON output is a straightforward "toString" on the txn object
302          if (firstTxn)
303            firstTxn = false;
304          else
305            out.print(",");
306          // encode and print JSON
307          out.print(MAPPER.writeValueAsString(txn));
308        } else {
309          // Pretty output, complete with indentation by atomic action
310          out.println("Sequence=" + txn.get("sequence") + " "
311              + ", region=" + txn.get("region") + " at write timestamp=" + new Date(writeTime));
312          for (int i = 0; i < actions.size(); i++) {
313            Map op = actions.get(i);
314            out.println("row=" + op.get("row") +
315                ", column=" + op.get("family") + ":" + op.get("qualifier"));
316            if (op.get("tag") != null) {
317              out.println("    tag: " + op.get("tag"));
318            }
319            if (outputValues) out.println("    value: " + op.get("value"));
320          }
321        }
322      }
323    } finally {
324      log.close();
325    }
326    if (outputJSON && !persistentOutput) {
327      out.print("]");
328    }
329  }
330
331  private static Map<String, Object> toStringMap(Cell cell) {
332    Map<String, Object> stringMap = new HashMap<>();
333    stringMap.put("row",
334        Bytes.toStringBinary(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength()));
335    stringMap.put("family", Bytes.toStringBinary(cell.getFamilyArray(), cell.getFamilyOffset(),
336                cell.getFamilyLength()));
337    stringMap.put("qualifier",
338        Bytes.toStringBinary(cell.getQualifierArray(), cell.getQualifierOffset(),
339            cell.getQualifierLength()));
340    stringMap.put("timestamp", cell.getTimestamp());
341    stringMap.put("vlen", cell.getValueLength());
342    if (cell.getTagsLength() > 0) {
343      List<String> tagsString = new ArrayList<>();
344      Iterator<Tag> tagsIterator = PrivateCellUtil.tagsIterator(cell);
345      while (tagsIterator.hasNext()) {
346        Tag tag = tagsIterator.next();
347        tagsString
348            .add((tag.getType()) + ":" + Bytes.toStringBinary(Tag.cloneValue(tag)));
349      }
350      stringMap.put("tag", tagsString);
351    }
352    return stringMap;
353  }
354
355  public static void main(String[] args) throws IOException {
356    run(args);
357  }
358
359  /**
360   * Pass one or more log file names and formatting options and it will dump out
361   * a text version of the contents on <code>stdout</code>.
362   *
363   * @param args
364   *          Command line arguments
365   * @throws IOException
366   *           Thrown upon file system errors etc.
367   */
368  public static void run(String[] args) throws IOException {
369    // create options
370    Options options = new Options();
371    options.addOption("h", "help", false, "Output help message");
372    options.addOption("j", "json", false, "Output JSON");
373    options.addOption("p", "printvals", false, "Print values");
374    options.addOption("r", "region", true,
375        "Region to filter by. Pass encoded region name; e.g. '9192caead6a5a20acb4454ffbc79fa14'");
376    options.addOption("s", "sequence", true,
377        "Sequence to filter by. Pass sequence number.");
378    options.addOption("w", "row", true, "Row to filter by. Pass row name.");
379
380    WALPrettyPrinter printer = new WALPrettyPrinter();
381    CommandLineParser parser = new PosixParser();
382    List<?> files = null;
383    try {
384      CommandLine cmd = parser.parse(options, args);
385      files = cmd.getArgList();
386      if (files.isEmpty() || cmd.hasOption("h")) {
387        HelpFormatter formatter = new HelpFormatter();
388        formatter.printHelp("WAL <filename...>", options, true);
389        System.exit(-1);
390      }
391      // configure the pretty printer using command line options
392      if (cmd.hasOption("p"))
393        printer.enableValues();
394      if (cmd.hasOption("j"))
395        printer.enableJSON();
396      if (cmd.hasOption("r"))
397        printer.setRegionFilter(cmd.getOptionValue("r"));
398      if (cmd.hasOption("s"))
399        printer.setSequenceFilter(Long.parseLong(cmd.getOptionValue("s")));
400      if (cmd.hasOption("w"))
401        printer.setRowFilter(cmd.getOptionValue("w"));
402    } catch (ParseException e) {
403      e.printStackTrace();
404      HelpFormatter formatter = new HelpFormatter();
405      formatter.printHelp("HFile filename(s) ", options, true);
406      System.exit(-1);
407    }
408    // get configuration, file system, and process the given files
409    Configuration conf = HBaseConfiguration.create();
410    FSUtils.setFsDefault(conf, FSUtils.getRootDir(conf));
411
412    // begin output
413    printer.beginPersistentOutput();
414    for (Object f : files) {
415      Path file = new Path((String) f);
416      FileSystem fs = file.getFileSystem(conf);
417      if (!fs.exists(file)) {
418        System.err.println("ERROR, file doesnt exist: " + file);
419        return;
420      }
421      printer.processFile(conf, file);
422    }
423    printer.endPersistentOutput();
424  }
425}