001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertNotNull;
022import static org.junit.Assert.assertTrue;
023import static org.junit.Assert.fail;
024
025import java.io.ByteArrayInputStream;
026import java.io.ByteArrayOutputStream;
027import java.io.DataInputStream;
028import java.io.DataOutputStream;
029import java.io.IOException;
030import java.util.List;
031import java.util.Map;
032import java.util.NavigableSet;
033import java.util.Set;
034import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
035import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
036import org.apache.hadoop.hbase.client.Get;
037import org.apache.hadoop.hbase.client.RegionInfo;
038import org.apache.hadoop.hbase.client.RegionInfoBuilder;
039import org.apache.hadoop.hbase.client.Scan;
040import org.apache.hadoop.hbase.client.TableDescriptor;
041import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
042import org.apache.hadoop.hbase.filter.BinaryComparator;
043import org.apache.hadoop.hbase.filter.Filter;
044import org.apache.hadoop.hbase.filter.PrefixFilter;
045import org.apache.hadoop.hbase.filter.RowFilter;
046import org.apache.hadoop.hbase.io.TimeRange;
047import org.apache.hadoop.hbase.testclassification.MiscTests;
048import org.apache.hadoop.hbase.testclassification.SmallTests;
049import org.apache.hadoop.hbase.util.Bytes;
050import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
051import org.apache.hadoop.io.DataInputBuffer;
052import org.junit.ClassRule;
053import org.junit.Test;
054import org.junit.experimental.categories.Category;
055
056import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
057import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos;
058
059/**
060 * Test HBase Writables serializations
061 */
062@Category({ MiscTests.class, SmallTests.class })
063public class TestSerialization {
064
065  @ClassRule
066  public static final HBaseClassTestRule CLASS_RULE =
067    HBaseClassTestRule.forClass(TestSerialization.class);
068
069  @Test
070  public void testKeyValue() throws Exception {
071    final String name = "testKeyValue2";
072    byte[] row = Bytes.toBytes(name);
073    byte[] fam = Bytes.toBytes("fam");
074    byte[] qf = Bytes.toBytes("qf");
075    long ts = EnvironmentEdgeManager.currentTime();
076    byte[] val = Bytes.toBytes("val");
077    KeyValue kv = new KeyValue(row, fam, qf, ts, val);
078    ByteArrayOutputStream baos = new ByteArrayOutputStream();
079    DataOutputStream dos = new DataOutputStream(baos);
080    KeyValueUtil.write(kv, dos);
081    dos.close();
082    byte[] mb = baos.toByteArray();
083    ByteArrayInputStream bais = new ByteArrayInputStream(mb);
084    DataInputStream dis = new DataInputStream(bais);
085    KeyValue deserializedKv = KeyValueUtil.create(dis);
086    assertTrue(Bytes.equals(kv.getBuffer(), deserializedKv.getBuffer()));
087    assertEquals(kv.getOffset(), deserializedKv.getOffset());
088    assertEquals(kv.getLength(), deserializedKv.getLength());
089  }
090
091  @Test
092  public void testCreateKeyValueInvalidNegativeLength() {
093
094    KeyValue kv_0 = new KeyValue(Bytes.toBytes("myRow"), Bytes.toBytes("myCF"), // 51 bytes
095      Bytes.toBytes("myQualifier"), 12345L, Bytes.toBytes("my12345"));
096
097    KeyValue kv_1 = new KeyValue(Bytes.toBytes("myRow"), Bytes.toBytes("myCF"), // 49 bytes
098      Bytes.toBytes("myQualifier"), 12345L, Bytes.toBytes("my123"));
099
100    ByteArrayOutputStream baos = new ByteArrayOutputStream();
101    DataOutputStream dos = new DataOutputStream(baos);
102
103    long l = 0;
104    try {
105      l = KeyValue.oswrite(kv_0, dos, false);
106      l += KeyValue.oswrite(kv_1, dos, false);
107      assertEquals(100L, l);
108    } catch (IOException e) {
109      fail("Unexpected IOException" + e.getMessage());
110    }
111
112    ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
113    DataInputStream dis = new DataInputStream(bais);
114
115    try {
116      KeyValueUtil.create(dis);
117      assertTrue(kv_0.equals(kv_1));
118    } catch (Exception e) {
119      fail("Unexpected Exception" + e.getMessage());
120    }
121
122    // length -1
123    try {
124      // even if we have a good kv now in dis we will just pass length with -1 for simplicity
125      KeyValueUtil.create(-1, dis);
126      fail("Expected corrupt stream");
127    } catch (Exception e) {
128      assertEquals("Failed read -1 bytes, stream corrupt?", e.getMessage());
129    }
130
131  }
132
133  @Test
134  public void testCompareFilter() throws Exception {
135    Filter f =
136      new RowFilter(CompareOperator.EQUAL, new BinaryComparator(Bytes.toBytes("testRowOne-2")));
137    byte[] bytes = f.toByteArray();
138    Filter ff = RowFilter.parseFrom(bytes);
139    assertNotNull(ff);
140  }
141
142  @Test
143  public void testTableDescriptor() throws Exception {
144    final String name = "testTableDescriptor";
145    TableDescriptor htd = createTableDescriptor(name);
146    byte[] mb = TableDescriptorBuilder.toByteArray(htd);
147    TableDescriptor deserializedHtd = TableDescriptorBuilder.parseFrom(mb);
148    assertEquals(htd.getTableName(), deserializedHtd.getTableName());
149  }
150
151  /**
152   * Test RegionInfo serialization
153   * @throws Exception
154   */
155  @Test
156  public void testRegionInfo() throws Exception {
157    RegionInfo hri = createRandomRegion("testRegionInfo");
158
159    // test toByteArray()
160    byte[] hrib = RegionInfo.toByteArray(hri);
161    RegionInfo deserializedHri = RegionInfo.parseFrom(hrib);
162    assertEquals(hri.getEncodedName(), deserializedHri.getEncodedName());
163    assertEquals(hri, deserializedHri);
164
165    // test toDelimitedByteArray()
166    hrib = RegionInfo.toDelimitedByteArray(hri);
167    DataInputBuffer buf = new DataInputBuffer();
168    try {
169      buf.reset(hrib, hrib.length);
170      deserializedHri = RegionInfo.parseFrom(buf);
171      assertEquals(hri.getEncodedName(), deserializedHri.getEncodedName());
172      assertEquals(hri, deserializedHri);
173    } finally {
174      buf.close();
175    }
176  }
177
178  @Test
179  public void testRegionInfos() throws Exception {
180    RegionInfo hri = createRandomRegion("testRegionInfos");
181    byte[] triple = RegionInfo.toDelimitedByteArray(hri, hri, hri);
182    List<RegionInfo> regions = RegionInfo.parseDelimitedFrom(triple, 0, triple.length);
183    assertTrue(regions.size() == 3);
184    assertTrue(regions.get(0).equals(regions.get(1)));
185    assertTrue(regions.get(0).equals(regions.get(2)));
186  }
187
188  private RegionInfo createRandomRegion(final String name) {
189    TableDescriptorBuilder tableDescriptorBuilder =
190      TableDescriptorBuilder.newBuilder(TableName.valueOf(name));
191    String[] families = new String[] { "info", "anchor" };
192    for (int i = 0; i < families.length; i++) {
193      ColumnFamilyDescriptor columnFamilyDescriptor =
194        ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(families[i])).build();
195      tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor);
196    }
197    TableDescriptor tableDescriptor = tableDescriptorBuilder.build();
198    return RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build();
199  }
200
201  @Test
202  public void testGet() throws Exception {
203    byte[] row = Bytes.toBytes("row");
204    byte[] fam = Bytes.toBytes("fam");
205    byte[] qf1 = Bytes.toBytes("qf1");
206    long ts = EnvironmentEdgeManager.currentTime();
207    int maxVersions = 2;
208
209    Get get = new Get(row);
210    get.addColumn(fam, qf1);
211    get.setTimeRange(ts, ts + 1);
212    get.readVersions(maxVersions);
213
214    ClientProtos.Get getProto = ProtobufUtil.toGet(get);
215    Get desGet = ProtobufUtil.toGet(getProto);
216
217    assertTrue(Bytes.equals(get.getRow(), desGet.getRow()));
218    Set<byte[]> set = null;
219    Set<byte[]> desSet = null;
220
221    for (Map.Entry<byte[], NavigableSet<byte[]>> entry : get.getFamilyMap().entrySet()) {
222      assertTrue(desGet.getFamilyMap().containsKey(entry.getKey()));
223      set = entry.getValue();
224      desSet = desGet.getFamilyMap().get(entry.getKey());
225      for (byte[] qualifier : set) {
226        assertTrue(desSet.contains(qualifier));
227      }
228    }
229
230    assertEquals(get.getMaxVersions(), desGet.getMaxVersions());
231    TimeRange tr = get.getTimeRange();
232    TimeRange desTr = desGet.getTimeRange();
233    assertEquals(tr.getMax(), desTr.getMax());
234    assertEquals(tr.getMin(), desTr.getMin());
235  }
236
237  @Test
238  public void testScan() throws Exception {
239
240    byte[] startRow = Bytes.toBytes("startRow");
241    byte[] stopRow = Bytes.toBytes("stopRow");
242    byte[] fam = Bytes.toBytes("fam");
243    byte[] qf1 = Bytes.toBytes("qf1");
244    long ts = EnvironmentEdgeManager.currentTime();
245    int maxVersions = 2;
246
247    Scan scan = new Scan().withStartRow(startRow).withStopRow(stopRow);
248    scan.addColumn(fam, qf1);
249    scan.setTimeRange(ts, ts + 1);
250    scan.readVersions(maxVersions);
251
252    ClientProtos.Scan scanProto = ProtobufUtil.toScan(scan);
253    Scan desScan = ProtobufUtil.toScan(scanProto);
254
255    assertTrue(Bytes.equals(scan.getStartRow(), desScan.getStartRow()));
256    assertTrue(Bytes.equals(scan.getStopRow(), desScan.getStopRow()));
257    assertEquals(scan.getCacheBlocks(), desScan.getCacheBlocks());
258    Set<byte[]> set = null;
259    Set<byte[]> desSet = null;
260
261    for (Map.Entry<byte[], NavigableSet<byte[]>> entry : scan.getFamilyMap().entrySet()) {
262      assertTrue(desScan.getFamilyMap().containsKey(entry.getKey()));
263      set = entry.getValue();
264      desSet = desScan.getFamilyMap().get(entry.getKey());
265      for (byte[] column : set) {
266        assertTrue(desSet.contains(column));
267      }
268
269      // Test filters are serialized properly.
270      scan = new Scan().withStartRow(startRow);
271      final String name = "testScan";
272      byte[] prefix = Bytes.toBytes(name);
273      scan.setFilter(new PrefixFilter(prefix));
274      scanProto = ProtobufUtil.toScan(scan);
275      desScan = ProtobufUtil.toScan(scanProto);
276      Filter f = desScan.getFilter();
277      assertTrue(f instanceof PrefixFilter);
278    }
279
280    assertEquals(scan.getMaxVersions(), desScan.getMaxVersions());
281    TimeRange tr = scan.getTimeRange();
282    TimeRange desTr = desScan.getTimeRange();
283    assertEquals(tr.getMax(), desTr.getMax());
284    assertEquals(tr.getMin(), desTr.getMin());
285  }
286
287  protected static final int MAXVERSIONS = 3;
288  protected final static byte[] fam1 = Bytes.toBytes("colfamily1");
289  protected final static byte[] fam2 = Bytes.toBytes("colfamily2");
290  protected final static byte[] fam3 = Bytes.toBytes("colfamily3");
291  protected static final byte[][] COLUMNS = { fam1, fam2, fam3 };
292
293  /**
294   * Create a table of name <code>name</code> with {@link #COLUMNS} for families.
295   * @param name Name to give table.
296   * @return Column descriptor.
297   */
298  protected TableDescriptor createTableDescriptor(final String name) {
299    return createTableDescriptor(name, MAXVERSIONS);
300  }
301
302  /**
303   * Create a table of name <code>name</code> with {@link #COLUMNS} for families.
304   * @param name Name to give table.
305   * @param versions How many versions to allow per column.
306   * @return Column descriptor.
307   */
308  protected TableDescriptor createTableDescriptor(final String name, final int versions) {
309    TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(TableName.valueOf(name));
310    builder
311      .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(fam1).setMaxVersions(versions)
312        .setBlockCacheEnabled(false).build())
313      .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(fam2).setMaxVersions(versions)
314        .setBlockCacheEnabled(false).build())
315      .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(fam3).setMaxVersions(versions)
316        .setBlockCacheEnabled(false).build());
317    return builder.build();
318  }
319}