001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.ipc;
019
020import static org.junit.Assert.assertEquals;
021
022import java.io.IOException;
023import java.nio.ByteBuffer;
024import org.apache.commons.lang3.time.StopWatch;
025import org.apache.hadoop.hbase.Cell;
026import org.apache.hadoop.hbase.ExtendedCell;
027import org.apache.hadoop.hbase.ExtendedCellScanner;
028import org.apache.hadoop.hbase.HBaseClassTestRule;
029import org.apache.hadoop.hbase.HBaseConfiguration;
030import org.apache.hadoop.hbase.KeyValue;
031import org.apache.hadoop.hbase.PrivateCellUtil;
032import org.apache.hadoop.hbase.codec.Codec;
033import org.apache.hadoop.hbase.codec.KeyValueCodec;
034import org.apache.hadoop.hbase.io.SizedExtendedCellScanner;
035import org.apache.hadoop.hbase.nio.SingleByteBuff;
036import org.apache.hadoop.hbase.testclassification.ClientTests;
037import org.apache.hadoop.hbase.testclassification.SmallTests;
038import org.apache.hadoop.hbase.util.Bytes;
039import org.apache.hadoop.hbase.util.ClassSize;
040import org.apache.hadoop.io.compress.CompressionCodec;
041import org.apache.hadoop.io.compress.DefaultCodec;
042import org.apache.hadoop.io.compress.GzipCodec;
043import org.junit.Before;
044import org.junit.ClassRule;
045import org.junit.Test;
046import org.junit.experimental.categories.Category;
047import org.slf4j.Logger;
048import org.slf4j.LoggerFactory;
049
050@Category({ ClientTests.class, SmallTests.class })
051public class TestCellBlockBuilder {
052  @ClassRule
053  public static final HBaseClassTestRule CLASS_RULE =
054    HBaseClassTestRule.forClass(TestCellBlockBuilder.class);
055
056  private static final Logger LOG = LoggerFactory.getLogger(TestCellBlockBuilder.class);
057
058  private CellBlockBuilder builder;
059
060  @Before
061  public void before() {
062    this.builder = new CellBlockBuilder(HBaseConfiguration.create());
063  }
064
065  @Test
066  public void testBuildCellBlock() throws IOException {
067    doBuildCellBlockUndoCellBlock(this.builder, new KeyValueCodec(), null);
068    doBuildCellBlockUndoCellBlock(this.builder, new KeyValueCodec(), new DefaultCodec());
069    doBuildCellBlockUndoCellBlock(this.builder, new KeyValueCodec(), new GzipCodec());
070  }
071
072  static void doBuildCellBlockUndoCellBlock(final CellBlockBuilder builder, final Codec codec,
073    final CompressionCodec compressor) throws IOException {
074    doBuildCellBlockUndoCellBlock(builder, codec, compressor, 10, 1, false);
075  }
076
077  static void doBuildCellBlockUndoCellBlock(final CellBlockBuilder builder, final Codec codec,
078    final CompressionCodec compressor, final int count, final int size, final boolean sized)
079    throws IOException {
080    ExtendedCell[] cells = getCells(count, size);
081    ExtendedCellScanner cellScanner =
082      sized ? getSizedCellScanner(cells) : PrivateCellUtil.createExtendedCellScanner(cells);
083    ByteBuffer bb = builder.buildCellBlock(codec, compressor, cellScanner);
084    cellScanner =
085      builder.createCellScannerReusingBuffers(codec, compressor, new SingleByteBuff(bb));
086    int i = 0;
087    while (cellScanner.advance()) {
088      i++;
089    }
090    assertEquals(count, i);
091  }
092
093  static ExtendedCellScanner getSizedCellScanner(final ExtendedCell[] cells) {
094    int size = -1;
095    for (Cell cell : cells) {
096      size += PrivateCellUtil.estimatedSerializedSizeOf(cell);
097    }
098    final int totalSize = ClassSize.align(size);
099    final ExtendedCellScanner cellScanner = PrivateCellUtil.createExtendedCellScanner(cells);
100    return new SizedExtendedCellScanner() {
101      @Override
102      public long heapSize() {
103        return totalSize;
104      }
105
106      @Override
107      public ExtendedCell current() {
108        return cellScanner.current();
109      }
110
111      @Override
112      public boolean advance() throws IOException {
113        return cellScanner.advance();
114      }
115    };
116  }
117
118  static ExtendedCell[] getCells(final int howMany) {
119    return getCells(howMany, 1024);
120  }
121
122  static ExtendedCell[] getCells(final int howMany, final int valueSize) {
123    ExtendedCell[] cells = new ExtendedCell[howMany];
124    byte[] value = new byte[valueSize];
125    for (int i = 0; i < howMany; i++) {
126      byte[] index = Bytes.toBytes(i);
127      KeyValue kv = new KeyValue(index, Bytes.toBytes("f"), index, value);
128      cells[i] = kv;
129    }
130    return cells;
131  }
132
133  private static final String COUNT = "--count=";
134  private static final String SIZE = "--size=";
135
136  /**
137   * Prints usage and then exits w/ passed <code>errCode</code>
138   * @param errorCode the error code to use to exit the application
139   */
140  private static void usage(final int errorCode) {
141    System.out.println("Usage: IPCUtil [options]");
142    System.out.println("Micro-benchmarking how changed sizes and counts work with buffer resizing");
143    System.out.println(" --count  Count of Cells");
144    System.out.println(" --size   Size of Cell values");
145    System.out.println("Example: IPCUtil --count=1024 --size=1024");
146    System.exit(errorCode);
147  }
148
149  private static void timerTests(final CellBlockBuilder builder, final int count, final int size,
150    final Codec codec, final CompressionCodec compressor) throws IOException {
151    final int cycles = 1000;
152    StopWatch timer = new StopWatch();
153    timer.start();
154    for (int i = 0; i < cycles; i++) {
155      timerTest(builder, count, size, codec, compressor, false);
156    }
157    timer.stop();
158    LOG.info("Codec=" + codec + ", compression=" + compressor + ", sized=" + false + ", count="
159      + count + ", size=" + size + ", + took=" + timer.getTime() + "ms");
160    timer.reset();
161    timer.start();
162    for (int i = 0; i < cycles; i++) {
163      timerTest(builder, count, size, codec, compressor, true);
164    }
165    timer.stop();
166    LOG.info("Codec=" + codec + ", compression=" + compressor + ", sized=" + true + ", count="
167      + count + ", size=" + size + ", + took=" + timer.getTime() + "ms");
168  }
169
170  private static void timerTest(final CellBlockBuilder builder, final int count, final int size,
171    final Codec codec, final CompressionCodec compressor, final boolean sized) throws IOException {
172    doBuildCellBlockUndoCellBlock(builder, codec, compressor, count, size, sized);
173  }
174
175  /**
176   * For running a few tests of methods herein.
177   * @param args the arguments to use for the timer test
178   * @throws IOException if creating the build fails
179   */
180  public static void main(String[] args) throws IOException {
181    int count = 1024;
182    int size = 10240;
183    for (String arg : args) {
184      if (arg.startsWith(COUNT)) {
185        count = Integer.parseInt(arg.replace(COUNT, ""));
186      } else if (arg.startsWith(SIZE)) {
187        size = Integer.parseInt(arg.replace(SIZE, ""));
188      } else {
189        usage(1);
190      }
191    }
192    CellBlockBuilder builder = new CellBlockBuilder(HBaseConfiguration.create());
193    timerTests(builder, count, size, new KeyValueCodec(), null);
194    timerTests(builder, count, size, new KeyValueCodec(), new DefaultCodec());
195    timerTests(builder, count, size, new KeyValueCodec(), new GzipCodec());
196  }
197}