001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.ipc;
019
020import static org.junit.Assert.assertEquals;
021
022import java.io.IOException;
023import java.nio.ByteBuffer;
024import java.util.Arrays;
025import org.apache.commons.lang3.time.StopWatch;
026import org.apache.hadoop.hbase.Cell;
027import org.apache.hadoop.hbase.CellScanner;
028import org.apache.hadoop.hbase.CellUtil;
029import org.apache.hadoop.hbase.HBaseClassTestRule;
030import org.apache.hadoop.hbase.HBaseConfiguration;
031import org.apache.hadoop.hbase.KeyValue;
032import org.apache.hadoop.hbase.PrivateCellUtil;
033import org.apache.hadoop.hbase.codec.Codec;
034import org.apache.hadoop.hbase.codec.KeyValueCodec;
035import org.apache.hadoop.hbase.io.SizedCellScanner;
036import org.apache.hadoop.hbase.nio.SingleByteBuff;
037import org.apache.hadoop.hbase.testclassification.ClientTests;
038import org.apache.hadoop.hbase.testclassification.SmallTests;
039import org.apache.hadoop.hbase.util.Bytes;
040import org.apache.hadoop.hbase.util.ClassSize;
041import org.apache.hadoop.io.compress.CompressionCodec;
042import org.apache.hadoop.io.compress.DefaultCodec;
043import org.apache.hadoop.io.compress.GzipCodec;
044import org.junit.Before;
045import org.junit.ClassRule;
046import org.junit.Test;
047import org.junit.experimental.categories.Category;
048import org.slf4j.Logger;
049import org.slf4j.LoggerFactory;
050
051@Category({ ClientTests.class, SmallTests.class })
052public class TestCellBlockBuilder {
053  @ClassRule
054  public static final HBaseClassTestRule CLASS_RULE =
055      HBaseClassTestRule.forClass(TestCellBlockBuilder.class);
056
057  private static final Logger LOG = LoggerFactory.getLogger(TestCellBlockBuilder.class);
058
059  private CellBlockBuilder builder;
060
061  @Before
062  public void before() {
063    this.builder = new CellBlockBuilder(HBaseConfiguration.create());
064  }
065
066  @Test
067  public void testBuildCellBlock() throws IOException {
068    doBuildCellBlockUndoCellBlock(this.builder, new KeyValueCodec(), null);
069    doBuildCellBlockUndoCellBlock(this.builder, new KeyValueCodec(), new DefaultCodec());
070    doBuildCellBlockUndoCellBlock(this.builder, new KeyValueCodec(), new GzipCodec());
071  }
072
073  static void doBuildCellBlockUndoCellBlock(final CellBlockBuilder builder, final Codec codec,
074      final CompressionCodec compressor) throws IOException {
075    doBuildCellBlockUndoCellBlock(builder, codec, compressor, 10, 1, false);
076  }
077
078  static void doBuildCellBlockUndoCellBlock(final CellBlockBuilder builder, final Codec codec,
079      final CompressionCodec compressor, final int count, final int size, final boolean sized)
080      throws IOException {
081    Cell[] cells = getCells(count, size);
082    CellScanner cellScanner = sized ? getSizedCellScanner(cells)
083        : CellUtil.createCellScanner(Arrays.asList(cells).iterator());
084    ByteBuffer bb = builder.buildCellBlock(codec, compressor, cellScanner);
085    cellScanner = builder.createCellScannerReusingBuffers(codec, compressor,
086        new SingleByteBuff(bb));
087    int i = 0;
088    while (cellScanner.advance()) {
089      i++;
090    }
091    assertEquals(count, i);
092  }
093
094  static CellScanner getSizedCellScanner(final Cell[] cells) {
095    int size = -1;
096    for (Cell cell : cells) {
097      size += PrivateCellUtil.estimatedSerializedSizeOf(cell);
098    }
099    final int totalSize = ClassSize.align(size);
100    final CellScanner cellScanner = CellUtil.createCellScanner(cells);
101    return new SizedCellScanner() {
102      @Override
103      public long heapSize() {
104        return totalSize;
105      }
106
107      @Override
108      public Cell current() {
109        return cellScanner.current();
110      }
111
112      @Override
113      public boolean advance() throws IOException {
114        return cellScanner.advance();
115      }
116    };
117  }
118
119  static Cell[] getCells(final int howMany) {
120    return getCells(howMany, 1024);
121  }
122
123  static Cell[] getCells(final int howMany, final int valueSize) {
124    Cell[] cells = new Cell[howMany];
125    byte[] value = new byte[valueSize];
126    for (int i = 0; i < howMany; i++) {
127      byte[] index = Bytes.toBytes(i);
128      KeyValue kv = new KeyValue(index, Bytes.toBytes("f"), index, value);
129      cells[i] = kv;
130    }
131    return cells;
132  }
133
134  private static final String COUNT = "--count=";
135  private static final String SIZE = "--size=";
136
137  /**
138   * Prints usage and then exits w/ passed <code>errCode</code>
139   * @param errorCode the error code to use to exit the application
140   */
141  private static void usage(final int errorCode) {
142    System.out.println("Usage: IPCUtil [options]");
143    System.out.println("Micro-benchmarking how changed sizes and counts work with buffer resizing");
144    System.out.println(" --count  Count of Cells");
145    System.out.println(" --size   Size of Cell values");
146    System.out.println("Example: IPCUtil --count=1024 --size=1024");
147    System.exit(errorCode);
148  }
149
150  private static void timerTests(final CellBlockBuilder builder, final int count, final int size,
151      final Codec codec, final CompressionCodec compressor) throws IOException {
152    final int cycles = 1000;
153    StopWatch timer = new StopWatch();
154    timer.start();
155    for (int i = 0; i < cycles; i++) {
156      timerTest(builder, timer, count, size, codec, compressor, false);
157    }
158    timer.stop();
159    LOG.info("Codec=" + codec + ", compression=" + compressor + ", sized=" + false + ", count="
160        + count + ", size=" + size + ", + took=" + timer.getTime() + "ms");
161    timer.reset();
162    timer.start();
163    for (int i = 0; i < cycles; i++) {
164      timerTest(builder, timer, count, size, codec, compressor, true);
165    }
166    timer.stop();
167    LOG.info("Codec=" + codec + ", compression=" + compressor + ", sized=" + true + ", count="
168        + count + ", size=" + size + ", + took=" + timer.getTime() + "ms");
169  }
170
171  private static void timerTest(final CellBlockBuilder builder, final StopWatch timer,
172      final int count, final int size, final Codec codec, final CompressionCodec compressor,
173      final boolean sized) throws IOException {
174    doBuildCellBlockUndoCellBlock(builder, codec, compressor, count, size, sized);
175  }
176
177  /**
178   * For running a few tests of methods herein.
179   *
180   * @param args the arguments to use for the timer test
181   * @throws IOException if creating the build fails
182   */
183  public static void main(String[] args) throws IOException {
184    int count = 1024;
185    int size = 10240;
186    for (String arg : args) {
187      if (arg.startsWith(COUNT)) {
188        count = Integer.parseInt(arg.replace(COUNT, ""));
189      } else if (arg.startsWith(SIZE)) {
190        size = Integer.parseInt(arg.replace(SIZE, ""));
191      } else {
192        usage(1);
193      }
194    }
195    CellBlockBuilder builder = new CellBlockBuilder(HBaseConfiguration.create());
196    timerTests(builder, count, size, new KeyValueCodec(), null);
197    timerTests(builder, count, size, new KeyValueCodec(), new DefaultCodec());
198    timerTests(builder, count, size, new KeyValueCodec(), new GzipCodec());
199  }
200}