001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertNotNull;
022import static org.junit.Assert.assertTrue;
023import static org.junit.Assert.fail;
024
025import java.io.ByteArrayInputStream;
026import java.io.ByteArrayOutputStream;
027import java.io.DataInputStream;
028import java.io.DataOutputStream;
029import java.io.IOException;
030import java.util.List;
031import java.util.Map;
032import java.util.NavigableSet;
033import java.util.Set;
034import org.apache.hadoop.hbase.client.Get;
035import org.apache.hadoop.hbase.client.Scan;
036import org.apache.hadoop.hbase.filter.BinaryComparator;
037import org.apache.hadoop.hbase.filter.Filter;
038import org.apache.hadoop.hbase.filter.PrefixFilter;
039import org.apache.hadoop.hbase.filter.RowFilter;
040import org.apache.hadoop.hbase.io.TimeRange;
041import org.apache.hadoop.hbase.testclassification.MiscTests;
042import org.apache.hadoop.hbase.testclassification.SmallTests;
043import org.apache.hadoop.hbase.util.Bytes;
044import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
045import org.apache.hadoop.io.DataInputBuffer;
046import org.junit.ClassRule;
047import org.junit.Test;
048import org.junit.experimental.categories.Category;
049
050import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
051import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos;
052
053/**
054 * Test HBase Writables serializations
055 */
056@Category({ MiscTests.class, SmallTests.class })
057public class TestSerialization {
058
059  @ClassRule
060  public static final HBaseClassTestRule CLASS_RULE =
061    HBaseClassTestRule.forClass(TestSerialization.class);
062
063  @Test
064  public void testKeyValue() throws Exception {
065    final String name = "testKeyValue2";
066    byte[] row = name.getBytes();
067    byte[] fam = "fam".getBytes();
068    byte[] qf = "qf".getBytes();
069    long ts = EnvironmentEdgeManager.currentTime();
070    byte[] val = "val".getBytes();
071    KeyValue kv = new KeyValue(row, fam, qf, ts, val);
072    ByteArrayOutputStream baos = new ByteArrayOutputStream();
073    DataOutputStream dos = new DataOutputStream(baos);
074    long l = KeyValueUtil.write(kv, dos);
075    dos.close();
076    byte[] mb = baos.toByteArray();
077    ByteArrayInputStream bais = new ByteArrayInputStream(mb);
078    DataInputStream dis = new DataInputStream(bais);
079    KeyValue deserializedKv = KeyValueUtil.create(dis);
080    assertTrue(Bytes.equals(kv.getBuffer(), deserializedKv.getBuffer()));
081    assertEquals(kv.getOffset(), deserializedKv.getOffset());
082    assertEquals(kv.getLength(), deserializedKv.getLength());
083  }
084
085  @Test
086  public void testCreateKeyValueInvalidNegativeLength() {
087
088    KeyValue kv_0 = new KeyValue(Bytes.toBytes("myRow"), Bytes.toBytes("myCF"), // 51 bytes
089      Bytes.toBytes("myQualifier"), 12345L, Bytes.toBytes("my12345"));
090
091    KeyValue kv_1 = new KeyValue(Bytes.toBytes("myRow"), Bytes.toBytes("myCF"), // 49 bytes
092      Bytes.toBytes("myQualifier"), 12345L, Bytes.toBytes("my123"));
093
094    ByteArrayOutputStream baos = new ByteArrayOutputStream();
095    DataOutputStream dos = new DataOutputStream(baos);
096
097    long l = 0;
098    try {
099      l = KeyValue.oswrite(kv_0, dos, false);
100      l += KeyValue.oswrite(kv_1, dos, false);
101      assertEquals(100L, l);
102    } catch (IOException e) {
103      fail("Unexpected IOException" + e.getMessage());
104    }
105
106    ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
107    DataInputStream dis = new DataInputStream(bais);
108
109    try {
110      KeyValueUtil.create(dis);
111      assertTrue(kv_0.equals(kv_1));
112    } catch (Exception e) {
113      fail("Unexpected Exception" + e.getMessage());
114    }
115
116    // length -1
117    try {
118      // even if we have a good kv now in dis we will just pass length with -1 for simplicity
119      KeyValueUtil.create(-1, dis);
120      fail("Expected corrupt stream");
121    } catch (Exception e) {
122      assertEquals("Failed read -1 bytes, stream corrupt?", e.getMessage());
123    }
124
125  }
126
127  @Test
128  public void testCompareFilter() throws Exception {
129    Filter f =
130      new RowFilter(CompareOperator.EQUAL, new BinaryComparator(Bytes.toBytes("testRowOne-2")));
131    byte[] bytes = f.toByteArray();
132    Filter ff = RowFilter.parseFrom(bytes);
133    assertNotNull(ff);
134  }
135
136  @Test
137  public void testTableDescriptor() throws Exception {
138    final String name = "testTableDescriptor";
139    HTableDescriptor htd = createTableDescriptor(name);
140    byte[] mb = htd.toByteArray();
141    HTableDescriptor deserializedHtd = HTableDescriptor.parseFrom(mb);
142    assertEquals(htd.getTableName(), deserializedHtd.getTableName());
143  }
144
145  /**
146   * Test RegionInfo serialization n
147   */
148  @Test
149  public void testRegionInfo() throws Exception {
150    HRegionInfo hri = createRandomRegion("testRegionInfo");
151
152    // test toByteArray()
153    byte[] hrib = hri.toByteArray();
154    HRegionInfo deserializedHri = HRegionInfo.parseFrom(hrib);
155    assertEquals(hri.getEncodedName(), deserializedHri.getEncodedName());
156    assertEquals(hri, deserializedHri);
157
158    // test toDelimitedByteArray()
159    hrib = hri.toDelimitedByteArray();
160    DataInputBuffer buf = new DataInputBuffer();
161    try {
162      buf.reset(hrib, hrib.length);
163      deserializedHri = HRegionInfo.parseFrom(buf);
164      assertEquals(hri.getEncodedName(), deserializedHri.getEncodedName());
165      assertEquals(hri, deserializedHri);
166    } finally {
167      buf.close();
168    }
169  }
170
171  @Test
172  public void testRegionInfos() throws Exception {
173    HRegionInfo hri = createRandomRegion("testRegionInfos");
174    byte[] triple = HRegionInfo.toDelimitedByteArray(hri, hri, hri);
175    List<HRegionInfo> regions = HRegionInfo.parseDelimitedFrom(triple, 0, triple.length);
176    assertTrue(regions.size() == 3);
177    assertTrue(regions.get(0).equals(regions.get(1)));
178    assertTrue(regions.get(0).equals(regions.get(2)));
179  }
180
181  private HRegionInfo createRandomRegion(final String name) {
182    HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name));
183    String[] families = new String[] { "info", "anchor" };
184    for (int i = 0; i < families.length; i++) {
185      htd.addFamily(new HColumnDescriptor(families[i]));
186    }
187    return new HRegionInfo(htd.getTableName(), HConstants.EMPTY_START_ROW,
188      HConstants.EMPTY_END_ROW);
189  }
190
191  @Test
192  public void testGet() throws Exception {
193    byte[] row = "row".getBytes();
194    byte[] fam = "fam".getBytes();
195    byte[] qf1 = "qf1".getBytes();
196    long ts = EnvironmentEdgeManager.currentTime();
197    int maxVersions = 2;
198
199    Get get = new Get(row);
200    get.addColumn(fam, qf1);
201    get.setTimeRange(ts, ts + 1);
202    get.setMaxVersions(maxVersions);
203
204    ClientProtos.Get getProto = ProtobufUtil.toGet(get);
205    Get desGet = ProtobufUtil.toGet(getProto);
206
207    assertTrue(Bytes.equals(get.getRow(), desGet.getRow()));
208    Set<byte[]> set = null;
209    Set<byte[]> desSet = null;
210
211    for (Map.Entry<byte[], NavigableSet<byte[]>> entry : get.getFamilyMap().entrySet()) {
212      assertTrue(desGet.getFamilyMap().containsKey(entry.getKey()));
213      set = entry.getValue();
214      desSet = desGet.getFamilyMap().get(entry.getKey());
215      for (byte[] qualifier : set) {
216        assertTrue(desSet.contains(qualifier));
217      }
218    }
219
220    assertEquals(get.getMaxVersions(), desGet.getMaxVersions());
221    TimeRange tr = get.getTimeRange();
222    TimeRange desTr = desGet.getTimeRange();
223    assertEquals(tr.getMax(), desTr.getMax());
224    assertEquals(tr.getMin(), desTr.getMin());
225  }
226
227  @Test
228  public void testScan() throws Exception {
229    byte[] startRow = "startRow".getBytes();
230    byte[] stopRow = "stopRow".getBytes();
231    byte[] fam = "fam".getBytes();
232    byte[] qf1 = "qf1".getBytes();
233    long ts = EnvironmentEdgeManager.currentTime();
234    int maxVersions = 2;
235
236    Scan scan = new Scan(startRow, stopRow);
237    scan.addColumn(fam, qf1);
238    scan.setTimeRange(ts, ts + 1);
239    scan.setMaxVersions(maxVersions);
240
241    ClientProtos.Scan scanProto = ProtobufUtil.toScan(scan);
242    Scan desScan = ProtobufUtil.toScan(scanProto);
243
244    assertTrue(Bytes.equals(scan.getStartRow(), desScan.getStartRow()));
245    assertTrue(Bytes.equals(scan.getStopRow(), desScan.getStopRow()));
246    assertEquals(scan.getCacheBlocks(), desScan.getCacheBlocks());
247    Set<byte[]> set = null;
248    Set<byte[]> desSet = null;
249
250    for (Map.Entry<byte[], NavigableSet<byte[]>> entry : scan.getFamilyMap().entrySet()) {
251      assertTrue(desScan.getFamilyMap().containsKey(entry.getKey()));
252      set = entry.getValue();
253      desSet = desScan.getFamilyMap().get(entry.getKey());
254      for (byte[] column : set) {
255        assertTrue(desSet.contains(column));
256      }
257
258      // Test filters are serialized properly.
259      scan = new Scan(startRow);
260      final String name = "testScan";
261      byte[] prefix = Bytes.toBytes(name);
262      scan.setFilter(new PrefixFilter(prefix));
263      scanProto = ProtobufUtil.toScan(scan);
264      desScan = ProtobufUtil.toScan(scanProto);
265      Filter f = desScan.getFilter();
266      assertTrue(f instanceof PrefixFilter);
267    }
268
269    assertEquals(scan.getMaxVersions(), desScan.getMaxVersions());
270    TimeRange tr = scan.getTimeRange();
271    TimeRange desTr = desScan.getTimeRange();
272    assertEquals(tr.getMax(), desTr.getMax());
273    assertEquals(tr.getMin(), desTr.getMin());
274  }
275
276  /*
277   * TODO
278   * @Test public void testResultEmpty() throws Exception { List<KeyValue> keys = new
279   * ArrayList<KeyValue>(); Result r = Result.newResult(keys); assertTrue(r.isEmpty()); byte [] rb =
280   * Writables.getBytes(r); Result deserializedR = (Result)Writables.getWritable(rb, new Result());
281   * assertTrue(deserializedR.isEmpty()); }
282   * @Test public void testResult() throws Exception { byte [] rowA = Bytes.toBytes("rowA"); byte []
283   * famA = Bytes.toBytes("famA"); byte [] qfA = Bytes.toBytes("qfA"); byte [] valueA =
284   * Bytes.toBytes("valueA"); byte [] rowB = Bytes.toBytes("rowB"); byte [] famB =
285   * Bytes.toBytes("famB"); byte [] qfB = Bytes.toBytes("qfB"); byte [] valueB =
286   * Bytes.toBytes("valueB"); KeyValue kvA = new KeyValue(rowA, famA, qfA, valueA); KeyValue kvB =
287   * new KeyValue(rowB, famB, qfB, valueB); Result result = Result.newResult(new KeyValue[]{kvA,
288   * kvB}); byte [] rb = Writables.getBytes(result); Result deResult =
289   * (Result)Writables.getWritable(rb, new Result());
290   * assertTrue("results are not equivalent, first key mismatch",
291   * result.raw()[0].equals(deResult.raw()[0]));
292   * assertTrue("results are not equivalent, second key mismatch",
293   * result.raw()[1].equals(deResult.raw()[1])); // Test empty Result Result r = new Result(); byte
294   * [] b = Writables.getBytes(r); Result deserialized = (Result)Writables.getWritable(b, new
295   * Result()); assertEquals(r.size(), deserialized.size()); }
296   * @Test public void testResultDynamicBuild() throws Exception { byte [] rowA =
297   * Bytes.toBytes("rowA"); byte [] famA = Bytes.toBytes("famA"); byte [] qfA =
298   * Bytes.toBytes("qfA"); byte [] valueA = Bytes.toBytes("valueA"); byte [] rowB =
299   * Bytes.toBytes("rowB"); byte [] famB = Bytes.toBytes("famB"); byte [] qfB =
300   * Bytes.toBytes("qfB"); byte [] valueB = Bytes.toBytes("valueB"); KeyValue kvA = new
301   * KeyValue(rowA, famA, qfA, valueA); KeyValue kvB = new KeyValue(rowB, famB, qfB, valueB); Result
302   * result = Result.newResult(new KeyValue[]{kvA, kvB}); byte [] rb = Writables.getBytes(result);
303   * // Call getRow() first Result deResult = (Result)Writables.getWritable(rb, new Result()); byte
304   * [] row = deResult.getRow(); assertTrue(Bytes.equals(row, rowA)); // Call sorted() first
305   * deResult = (Result)Writables.getWritable(rb, new Result());
306   * assertTrue("results are not equivalent, first key mismatch",
307   * result.raw()[0].equals(deResult.raw()[0]));
308   * assertTrue("results are not equivalent, second key mismatch",
309   * result.raw()[1].equals(deResult.raw()[1])); // Call raw() first deResult =
310   * (Result)Writables.getWritable(rb, new Result());
311   * assertTrue("results are not equivalent, first key mismatch",
312   * result.raw()[0].equals(deResult.raw()[0]));
313   * assertTrue("results are not equivalent, second key mismatch",
314   * result.raw()[1].equals(deResult.raw()[1])); }
315   * @Test public void testResultArray() throws Exception { byte [] rowA = Bytes.toBytes("rowA");
316   * byte [] famA = Bytes.toBytes("famA"); byte [] qfA = Bytes.toBytes("qfA"); byte [] valueA =
317   * Bytes.toBytes("valueA"); byte [] rowB = Bytes.toBytes("rowB"); byte [] famB =
318   * Bytes.toBytes("famB"); byte [] qfB = Bytes.toBytes("qfB"); byte [] valueB =
319   * Bytes.toBytes("valueB"); KeyValue kvA = new KeyValue(rowA, famA, qfA, valueA); KeyValue kvB =
320   * new KeyValue(rowB, famB, qfB, valueB); Result result1 = Result.newResult(new KeyValue[]{kvA,
321   * kvB}); Result result2 = Result.newResult(new KeyValue[]{kvB}); Result result3 =
322   * Result.newResult(new KeyValue[]{kvB}); Result [] results = new Result [] {result1, result2,
323   * result3}; ByteArrayOutputStream byteStream = new ByteArrayOutputStream(); DataOutputStream out
324   * = new DataOutputStream(byteStream); Result.writeArray(out, results); byte [] rb =
325   * byteStream.toByteArray(); DataInputBuffer in = new DataInputBuffer(); in.reset(rb, 0,
326   * rb.length); Result [] deResults = Result.readArray(in); assertTrue(results.length ==
327   * deResults.length); for(int i=0;i<results.length;i++) { KeyValue [] keysA = results[i].raw();
328   * KeyValue [] keysB = deResults[i].raw(); assertTrue(keysA.length == keysB.length); for(int
329   * j=0;j<keysA.length;j++) { assertTrue("Expected equivalent keys but found:\n" + "KeyA : " +
330   * keysA[j].toString() + "\n" + "KeyB : " + keysB[j].toString() + "\n" + keysA.length +
331   * " total keys, " + i + "th so far" ,keysA[j].equals(keysB[j])); } } }
332   * @Test public void testResultArrayEmpty() throws Exception { List<KeyValue> keys = new
333   * ArrayList<KeyValue>(); Result r = Result.newResult(keys); Result [] results = new Result []
334   * {r}; ByteArrayOutputStream byteStream = new ByteArrayOutputStream(); DataOutputStream out = new
335   * DataOutputStream(byteStream); Result.writeArray(out, results); results = null; byteStream = new
336   * ByteArrayOutputStream(); out = new DataOutputStream(byteStream); Result.writeArray(out,
337   * results); byte [] rb = byteStream.toByteArray(); DataInputBuffer in = new DataInputBuffer();
338   * in.reset(rb, 0, rb.length); Result [] deResults = Result.readArray(in);
339   * assertTrue(deResults.length == 0); results = new Result[0]; byteStream = new
340   * ByteArrayOutputStream(); out = new DataOutputStream(byteStream); Result.writeArray(out,
341   * results); rb = byteStream.toByteArray(); in = new DataInputBuffer(); in.reset(rb, 0,
342   * rb.length); deResults = Result.readArray(in); assertTrue(deResults.length == 0); }
343   */
344
345  protected static final int MAXVERSIONS = 3;
346  protected final static byte[] fam1 = Bytes.toBytes("colfamily1");
347  protected final static byte[] fam2 = Bytes.toBytes("colfamily2");
348  protected final static byte[] fam3 = Bytes.toBytes("colfamily3");
349  protected static final byte[][] COLUMNS = { fam1, fam2, fam3 };
350
351  /**
352   * Create a table of name <code>name</code> with {@link #COLUMNS} for families.
353   * @param name Name to give table.
354   * @return Column descriptor.
355   */
356  protected HTableDescriptor createTableDescriptor(final String name) {
357    return createTableDescriptor(name, MAXVERSIONS);
358  }
359
360  /**
361   * Create a table of name <code>name</code> with {@link #COLUMNS} for families.
362   * @param name     Name to give table.
363   * @param versions How many versions to allow per column.
364   * @return Column descriptor.
365   */
366  protected HTableDescriptor createTableDescriptor(final String name, final int versions) {
367    HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name));
368    htd.addFamily(new HColumnDescriptor(fam1).setMaxVersions(versions).setBlockCacheEnabled(false));
369    htd.addFamily(new HColumnDescriptor(fam2).setMaxVersions(versions).setBlockCacheEnabled(false));
370    htd.addFamily(new HColumnDescriptor(fam3).setMaxVersions(versions).setBlockCacheEnabled(false));
371    return htd;
372  }
373}