Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase;
019
020import static org.junit.jupiter.api.Assertions.assertEquals;
021import static org.junit.jupiter.api.Assertions.assertNotNull;
022import static org.junit.jupiter.api.Assertions.assertTrue;
023import static org.junit.jupiter.api.Assertions.fail;
024
025import java.io.ByteArrayInputStream;
026import java.io.ByteArrayOutputStream;
027import java.io.DataInputStream;
028import java.io.DataOutputStream;
029import java.io.IOException;
030import java.util.List;
031import java.util.Map;
032import java.util.NavigableSet;
033import java.util.Set;
034import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
035import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
036import org.apache.hadoop.hbase.client.Get;
037import org.apache.hadoop.hbase.client.RegionInfo;
038import org.apache.hadoop.hbase.client.RegionInfoBuilder;
039import org.apache.hadoop.hbase.client.Scan;
040import org.apache.hadoop.hbase.client.TableDescriptor;
041import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
042import org.apache.hadoop.hbase.filter.BinaryComparator;
043import org.apache.hadoop.hbase.filter.Filter;
044import org.apache.hadoop.hbase.filter.PrefixFilter;
045import org.apache.hadoop.hbase.filter.RowFilter;
046import org.apache.hadoop.hbase.io.TimeRange;
047import org.apache.hadoop.hbase.testclassification.MiscTests;
048import org.apache.hadoop.hbase.testclassification.SmallTests;
049import org.apache.hadoop.hbase.util.ByteBufferUtils;
050import org.apache.hadoop.hbase.util.Bytes;
051import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
052import org.apache.hadoop.io.DataInputBuffer;
053import org.junit.jupiter.api.Tag;
054import org.junit.jupiter.api.Test;
055
056import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
057import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos;
058
059/**
060 * Test HBase Writables serializations
061 */
062@Tag(MiscTests.TAG)
063@Tag(SmallTests.TAG)
064public class TestSerialization {
065
066  @Test
067  public void testKeyValue() throws Exception {
068    final String name = "testKeyValue2";
069    byte[] row = Bytes.toBytes(name);
070    byte[] fam = Bytes.toBytes("fam");
071    byte[] qf = Bytes.toBytes("qf");
072    long ts = EnvironmentEdgeManager.currentTime();
073    byte[] val = Bytes.toBytes("val");
074    KeyValue kv = new KeyValue(row, fam, qf, ts, val);
075    ByteArrayOutputStream baos = new ByteArrayOutputStream();
076    DataOutputStream dos = new DataOutputStream(baos);
077    KeyValueUtil.write(kv, dos);
078    dos.close();
079    byte[] mb = baos.toByteArray();
080    ByteArrayInputStream bais = new ByteArrayInputStream(mb);
081    DataInputStream dis = new DataInputStream(bais);
082    KeyValue deserializedKv = KeyValueUtil.create(dis);
083    assertTrue(Bytes.equals(kv.getBuffer(), deserializedKv.getBuffer()));
084    assertEquals(kv.getOffset(), deserializedKv.getOffset());
085    assertEquals(kv.getLength(), deserializedKv.getLength());
086  }
087
088  @Test
089  public void testCreateKeyValueInvalidNegativeLength() {
090
091    KeyValue kv_0 = new KeyValue(Bytes.toBytes("myRow"), Bytes.toBytes("myCF"), // 51 bytes
092      Bytes.toBytes("myQualifier"), 12345L, Bytes.toBytes("my12345"));
093
094    KeyValue kv_1 = new KeyValue(Bytes.toBytes("myRow"), Bytes.toBytes("myCF"), // 49 bytes
095      Bytes.toBytes("myQualifier"), 12345L, Bytes.toBytes("my123"));
096
097    ByteArrayOutputStream baos = new ByteArrayOutputStream();
098    DataOutputStream dos = new DataOutputStream(baos);
099
100    long l = 0;
101    try {
102      ByteBufferUtils.putInt(dos, kv_0.getSerializedSize(false));
103      l = (long) kv_0.write(dos, false) + Bytes.SIZEOF_INT;
104      ByteBufferUtils.putInt(dos, kv_1.getSerializedSize(false));
105      l += (long) kv_1.write(dos, false) + Bytes.SIZEOF_INT;
106      assertEquals(100L, l);
107    } catch (IOException e) {
108      fail("Unexpected IOException" + e.getMessage());
109    }
110
111    ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
112    DataInputStream dis = new DataInputStream(bais);
113
114    try {
115      KeyValueUtil.create(dis);
116      assertTrue(kv_0.equals(kv_1));
117    } catch (Exception e) {
118      fail("Unexpected Exception" + e.getMessage());
119    }
120
121    // length -1
122    try {
123      // even if we have a good kv now in dis we will just pass length with -1 for simplicity
124      KeyValueUtil.create(-1, dis);
125      fail("Expected corrupt stream");
126    } catch (Exception e) {
127      assertEquals("Failed read -1 bytes, stream corrupt?", e.getMessage());
128    }
129
130  }
131
132  @Test
133  public void testCompareFilter() throws Exception {
134    Filter f =
135      new RowFilter(CompareOperator.EQUAL, new BinaryComparator(Bytes.toBytes("testRowOne-2")));
136    byte[] bytes = f.toByteArray();
137    Filter ff = RowFilter.parseFrom(bytes);
138    assertNotNull(ff);
139  }
140
141  @Test
142  public void testTableDescriptor() throws Exception {
143    final String name = "testTableDescriptor";
144    TableDescriptor htd = createTableDescriptor(name);
145    byte[] mb = TableDescriptorBuilder.toByteArray(htd);
146    TableDescriptor deserializedHtd = TableDescriptorBuilder.parseFrom(mb);
147    assertEquals(htd.getTableName(), deserializedHtd.getTableName());
148  }
149
150  /**
151   * Test RegionInfo serialization
152   */
153  @Test
154  public void testRegionInfo() throws Exception {
155    RegionInfo hri = createRandomRegion("testRegionInfo");
156
157    // test toByteArray()
158    byte[] hrib = RegionInfo.toByteArray(hri);
159    RegionInfo deserializedHri = RegionInfo.parseFrom(hrib);
160    assertEquals(hri.getEncodedName(), deserializedHri.getEncodedName());
161    assertEquals(hri, deserializedHri);
162
163    // test toDelimitedByteArray()
164    hrib = RegionInfo.toDelimitedByteArray(hri);
165    DataInputBuffer buf = new DataInputBuffer();
166    try {
167      buf.reset(hrib, hrib.length);
168      deserializedHri = RegionInfo.parseFrom(buf);
169      assertEquals(hri.getEncodedName(), deserializedHri.getEncodedName());
170      assertEquals(hri, deserializedHri);
171    } finally {
172      buf.close();
173    }
174  }
175
176  @Test
177  public void testRegionInfos() throws Exception {
178    RegionInfo hri = createRandomRegion("testRegionInfos");
179    byte[] triple = RegionInfo.toDelimitedByteArray(hri, hri, hri);
180    List<RegionInfo> regions = RegionInfo.parseDelimitedFrom(triple, 0, triple.length);
181    assertTrue(regions.size() == 3);
182    assertTrue(regions.get(0).equals(regions.get(1)));
183    assertTrue(regions.get(0).equals(regions.get(2)));
184  }
185
186  private RegionInfo createRandomRegion(final String name) {
187    TableDescriptorBuilder tableDescriptorBuilder =
188      TableDescriptorBuilder.newBuilder(TableName.valueOf(name));
189    String[] families = new String[] { "info", "anchor" };
190    for (int i = 0; i < families.length; i++) {
191      ColumnFamilyDescriptor columnFamilyDescriptor =
192        ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(families[i])).build();
193      tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor);
194    }
195    TableDescriptor tableDescriptor = tableDescriptorBuilder.build();
196    return RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build();
197  }
198
199  @Test
200  public void testGet() throws Exception {
201    byte[] row = Bytes.toBytes("row");
202    byte[] fam = Bytes.toBytes("fam");
203    byte[] qf1 = Bytes.toBytes("qf1");
204    long ts = EnvironmentEdgeManager.currentTime();
205    int maxVersions = 2;
206
207    Get get = new Get(row);
208    get.addColumn(fam, qf1);
209    get.setTimeRange(ts, ts + 1);
210    get.readVersions(maxVersions);
211
212    ClientProtos.Get getProto = ProtobufUtil.toGet(get);
213    Get desGet = ProtobufUtil.toGet(getProto);
214
215    assertTrue(Bytes.equals(get.getRow(), desGet.getRow()));
216    Set<byte[]> set = null;
217    Set<byte[]> desSet = null;
218
219    for (Map.Entry<byte[], NavigableSet<byte[]>> entry : get.getFamilyMap().entrySet()) {
220      assertTrue(desGet.getFamilyMap().containsKey(entry.getKey()));
221      set = entry.getValue();
222      desSet = desGet.getFamilyMap().get(entry.getKey());
223      for (byte[] qualifier : set) {
224        assertTrue(desSet.contains(qualifier));
225      }
226    }
227
228    assertEquals(get.getMaxVersions(), desGet.getMaxVersions());
229    TimeRange tr = get.getTimeRange();
230    TimeRange desTr = desGet.getTimeRange();
231    assertEquals(tr.getMax(), desTr.getMax());
232    assertEquals(tr.getMin(), desTr.getMin());
233  }
234
235  @Test
236  public void testScan() throws Exception {
237
238    byte[] startRow = Bytes.toBytes("startRow");
239    byte[] stopRow = Bytes.toBytes("stopRow");
240    byte[] fam = Bytes.toBytes("fam");
241    byte[] qf1 = Bytes.toBytes("qf1");
242    long ts = EnvironmentEdgeManager.currentTime();
243    int maxVersions = 2;
244
245    Scan scan = new Scan().withStartRow(startRow).withStopRow(stopRow);
246    scan.addColumn(fam, qf1);
247    scan.setTimeRange(ts, ts + 1);
248    scan.readVersions(maxVersions);
249
250    ClientProtos.Scan scanProto = ProtobufUtil.toScan(scan);
251    Scan desScan = ProtobufUtil.toScan(scanProto);
252
253    assertTrue(Bytes.equals(scan.getStartRow(), desScan.getStartRow()));
254    assertTrue(Bytes.equals(scan.getStopRow(), desScan.getStopRow()));
255    assertEquals(scan.getCacheBlocks(), desScan.getCacheBlocks());
256    Set<byte[]> set = null;
257    Set<byte[]> desSet = null;
258
259    for (Map.Entry<byte[], NavigableSet<byte[]>> entry : scan.getFamilyMap().entrySet()) {
260      assertTrue(desScan.getFamilyMap().containsKey(entry.getKey()));
261      set = entry.getValue();
262      desSet = desScan.getFamilyMap().get(entry.getKey());
263      for (byte[] column : set) {
264        assertTrue(desSet.contains(column));
265      }
266
267      // Test filters are serialized properly.
268      scan = new Scan().withStartRow(startRow);
269      final String name = "testScan";
270      byte[] prefix = Bytes.toBytes(name);
271      scan.setFilter(new PrefixFilter(prefix));
272      scanProto = ProtobufUtil.toScan(scan);
273      desScan = ProtobufUtil.toScan(scanProto);
274      Filter f = desScan.getFilter();
275      assertTrue(f instanceof PrefixFilter);
276    }
277
278    assertEquals(scan.getMaxVersions(), desScan.getMaxVersions());
279    TimeRange tr = scan.getTimeRange();
280    TimeRange desTr = desScan.getTimeRange();
281    assertEquals(tr.getMax(), desTr.getMax());
282    assertEquals(tr.getMin(), desTr.getMin());
283  }
284
285  protected static final int MAXVERSIONS = 3;
286  protected final static byte[] fam1 = Bytes.toBytes("colfamily1");
287  protected final static byte[] fam2 = Bytes.toBytes("colfamily2");
288  protected final static byte[] fam3 = Bytes.toBytes("colfamily3");
289  protected static final byte[][] COLUMNS = { fam1, fam2, fam3 };
290
291  /**
292   * Create a table of name <code>name</code> with {@link #COLUMNS} for families.
293   * @param name Name to give table.
294   * @return Column descriptor.
295   */
296  protected TableDescriptor createTableDescriptor(final String name) {
297    return createTableDescriptor(name, MAXVERSIONS);
298  }
299
300  /**
301   * Create a table of name <code>name</code> with {@link #COLUMNS} for families.
302   * @param name     Name to give table.
303   * @param versions How many versions to allow per column.
304   * @return Column descriptor.
305   */
306  protected TableDescriptor createTableDescriptor(final String name, final int versions) {
307    TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(TableName.valueOf(name));
308    builder
309      .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(fam1).setMaxVersions(versions)
310        .setBlockCacheEnabled(false).build())
311      .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(fam2).setMaxVersions(versions)
312        .setBlockCacheEnabled(false).build())
313      .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(fam3).setMaxVersions(versions)
314        .setBlockCacheEnabled(false).build());
315    return builder.build();
316  }
317}