001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertNotNull;
022import static org.junit.Assert.assertTrue;
023import static org.junit.Assert.fail;
024
025import java.io.ByteArrayInputStream;
026import java.io.ByteArrayOutputStream;
027import java.io.DataInputStream;
028import java.io.DataOutputStream;
029import java.io.IOException;
030import java.util.List;
031import java.util.Map;
032import java.util.NavigableSet;
033import java.util.Set;
034import org.apache.hadoop.hbase.client.Get;
035import org.apache.hadoop.hbase.client.Scan;
036import org.apache.hadoop.hbase.filter.BinaryComparator;
037import org.apache.hadoop.hbase.filter.Filter;
038import org.apache.hadoop.hbase.filter.PrefixFilter;
039import org.apache.hadoop.hbase.filter.RowFilter;
040import org.apache.hadoop.hbase.io.TimeRange;
041import org.apache.hadoop.hbase.testclassification.MiscTests;
042import org.apache.hadoop.hbase.testclassification.SmallTests;
043import org.apache.hadoop.hbase.util.Bytes;
044import org.apache.hadoop.io.DataInputBuffer;
045import org.junit.ClassRule;
046import org.junit.Test;
047import org.junit.experimental.categories.Category;
048
049import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
050import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos;
051
052/**
053 * Test HBase Writables serializations
054 */
055@Category({MiscTests.class, SmallTests.class})
056public class TestSerialization {
057
058  @ClassRule
059  public static final HBaseClassTestRule CLASS_RULE =
060      HBaseClassTestRule.forClass(TestSerialization.class);
061
062  @Test public void testKeyValue() throws Exception {
063    final String name = "testKeyValue2";
064    byte[] row = Bytes.toBytes(name);
065    byte[] fam = Bytes.toBytes("fam");
066    byte[] qf = Bytes.toBytes("qf");
067    long ts = System.currentTimeMillis();
068    byte[] val = Bytes.toBytes("val");
069    KeyValue kv = new KeyValue(row, fam, qf, ts, val);
070    ByteArrayOutputStream baos = new ByteArrayOutputStream();
071    DataOutputStream dos = new DataOutputStream(baos);
072    long l = KeyValueUtil.write(kv, dos);
073    dos.close();
074    byte [] mb = baos.toByteArray();
075    ByteArrayInputStream bais = new ByteArrayInputStream(mb);
076    DataInputStream dis = new DataInputStream(bais);
077    KeyValue deserializedKv = KeyValueUtil.create(dis);
078    assertTrue(Bytes.equals(kv.getBuffer(), deserializedKv.getBuffer()));
079    assertEquals(kv.getOffset(), deserializedKv.getOffset());
080    assertEquals(kv.getLength(), deserializedKv.getLength());
081  }
082
083  @Test public void testCreateKeyValueInvalidNegativeLength() {
084
085    KeyValue kv_0 = new KeyValue(Bytes.toBytes("myRow"), Bytes.toBytes("myCF"),       // 51 bytes
086                                 Bytes.toBytes("myQualifier"), 12345L, Bytes.toBytes("my12345"));
087
088    KeyValue kv_1 = new KeyValue(Bytes.toBytes("myRow"), Bytes.toBytes("myCF"),       // 49 bytes
089                                 Bytes.toBytes("myQualifier"), 12345L, Bytes.toBytes("my123"));
090
091    ByteArrayOutputStream baos = new ByteArrayOutputStream();
092    DataOutputStream dos = new DataOutputStream(baos);
093
094    long l = 0;
095    try {
096      l  = KeyValue.oswrite(kv_0, dos, false);
097      l += KeyValue.oswrite(kv_1, dos, false);
098      assertEquals(100L, l);
099    } catch (IOException e) {
100      fail("Unexpected IOException" + e.getMessage());
101    }
102
103    ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
104    DataInputStream dis = new DataInputStream(bais);
105
106    try {
107      KeyValueUtil.create(dis);
108      assertTrue(kv_0.equals(kv_1));
109    } catch (Exception e) {
110      fail("Unexpected Exception" + e.getMessage());
111    }
112
113    // length -1
114    try {
115      // even if we have a good kv now in dis we will just pass length with -1 for simplicity
116      KeyValueUtil.create(-1, dis);
117      fail("Expected corrupt stream");
118    } catch (Exception e) {
119      assertEquals("Failed read -1 bytes, stream corrupt?", e.getMessage());
120    }
121
122  }
123
124  @Test public void testCompareFilter() throws Exception {
125    Filter f = new RowFilter(CompareOperator.EQUAL,
126      new BinaryComparator(Bytes.toBytes("testRowOne-2")));
127    byte [] bytes = f.toByteArray();
128    Filter ff = RowFilter.parseFrom(bytes);
129    assertNotNull(ff);
130  }
131
132  @Test public void testTableDescriptor() throws Exception {
133    final String name = "testTableDescriptor";
134    HTableDescriptor htd = createTableDescriptor(name);
135    byte [] mb = htd.toByteArray();
136    HTableDescriptor deserializedHtd = HTableDescriptor.parseFrom(mb);
137    assertEquals(htd.getTableName(), deserializedHtd.getTableName());
138  }
139
140  /**
141   * Test RegionInfo serialization
142   * @throws Exception
143   */
144  @Test public void testRegionInfo() throws Exception {
145    HRegionInfo hri = createRandomRegion("testRegionInfo");
146
147    //test toByteArray()
148    byte [] hrib = hri.toByteArray();
149    HRegionInfo deserializedHri = HRegionInfo.parseFrom(hrib);
150    assertEquals(hri.getEncodedName(), deserializedHri.getEncodedName());
151    assertEquals(hri, deserializedHri);
152
153    //test toDelimitedByteArray()
154    hrib = hri.toDelimitedByteArray();
155    DataInputBuffer buf = new DataInputBuffer();
156    try {
157      buf.reset(hrib, hrib.length);
158      deserializedHri = HRegionInfo.parseFrom(buf);
159      assertEquals(hri.getEncodedName(), deserializedHri.getEncodedName());
160      assertEquals(hri, deserializedHri);
161    } finally {
162      buf.close();
163    }
164  }
165
166  @Test public void testRegionInfos() throws Exception {
167    HRegionInfo hri = createRandomRegion("testRegionInfos");
168    byte[] triple = HRegionInfo.toDelimitedByteArray(hri, hri, hri);
169    List<HRegionInfo> regions = HRegionInfo.parseDelimitedFrom(triple, 0, triple.length);
170    assertTrue(regions.size() == 3);
171    assertTrue(regions.get(0).equals(regions.get(1)));
172    assertTrue(regions.get(0).equals(regions.get(2)));
173  }
174
175  private HRegionInfo createRandomRegion(final String name) {
176    HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name));
177    String [] families = new String [] {"info", "anchor"};
178    for (int i = 0; i < families.length; i++) {
179      htd.addFamily(new HColumnDescriptor(families[i]));
180    }
181    return new HRegionInfo(htd.getTableName(), HConstants.EMPTY_START_ROW,
182      HConstants.EMPTY_END_ROW);
183  }
184
185  /*
186   * TODO
187  @Test public void testPut() throws Exception{
188    byte[] row = "row".getBytes();
189    byte[] fam = "fam".getBytes();
190    byte[] qf1 = "qf1".getBytes();
191    byte[] qf2 = "qf2".getBytes();
192    byte[] qf3 = "qf3".getBytes();
193    byte[] qf4 = "qf4".getBytes();
194    byte[] qf5 = "qf5".getBytes();
195    byte[] qf6 = "qf6".getBytes();
196    byte[] qf7 = "qf7".getBytes();
197    byte[] qf8 = "qf8".getBytes();
198
199    long ts = System.currentTimeMillis();
200    byte[] val = "val".getBytes();
201
202    Put put = new Put(row);
203    put.setWriteToWAL(false);
204    put.add(fam, qf1, ts, val);
205    put.add(fam, qf2, ts, val);
206    put.add(fam, qf3, ts, val);
207    put.add(fam, qf4, ts, val);
208    put.add(fam, qf5, ts, val);
209    put.add(fam, qf6, ts, val);
210    put.add(fam, qf7, ts, val);
211    put.add(fam, qf8, ts, val);
212
213    byte[] sb = Writables.getBytes(put);
214    Put desPut = (Put)Writables.getWritable(sb, new Put());
215
216    //Timing test
217//    long start = System.nanoTime();
218//    desPut = (Put)Writables.getWritable(sb, new Put());
219//    long stop = System.nanoTime();
220//    System.out.println("timer " +(stop-start));
221
222    assertTrue(Bytes.equals(put.getRow(), desPut.getRow()));
223    List<KeyValue> list = null;
224    List<KeyValue> desList = null;
225    for(Map.Entry<byte[], List<KeyValue>> entry : put.getFamilyMap().entrySet()){
226      assertTrue(desPut.getFamilyMap().containsKey(entry.getKey()));
227      list = entry.getValue();
228      desList = desPut.getFamilyMap().get(entry.getKey());
229      for(int i=0; i<list.size(); i++){
230        assertTrue(list.get(i).equals(desList.get(i)));
231      }
232    }
233  }
234
235
236  @Test public void testPut2() throws Exception{
237    byte[] row = "testAbort,,1243116656250".getBytes();
238    byte[] fam = "historian".getBytes();
239    byte[] qf1 = "creation".getBytes();
240
241    long ts = 9223372036854775807L;
242    byte[] val = "dont-care".getBytes();
243
244    Put put = new Put(row);
245    put.add(fam, qf1, ts, val);
246
247    byte[] sb = Writables.getBytes(put);
248    Put desPut = (Put)Writables.getWritable(sb, new Put());
249
250    assertTrue(Bytes.equals(put.getRow(), desPut.getRow()));
251    List<KeyValue> list = null;
252    List<KeyValue> desList = null;
253    for(Map.Entry<byte[], List<KeyValue>> entry : put.getFamilyMap().entrySet()){
254      assertTrue(desPut.getFamilyMap().containsKey(entry.getKey()));
255      list = entry.getValue();
256      desList = desPut.getFamilyMap().get(entry.getKey());
257      for(int i=0; i<list.size(); i++){
258        assertTrue(list.get(i).equals(desList.get(i)));
259      }
260    }
261  }
262
263
264  @Test public void testDelete() throws Exception{
265    byte[] row = "row".getBytes();
266    byte[] fam = "fam".getBytes();
267    byte[] qf1 = "qf1".getBytes();
268
269    long ts = System.currentTimeMillis();
270
271    Delete delete = new Delete(row);
272    delete.deleteColumn(fam, qf1, ts);
273
274    byte[] sb = Writables.getBytes(delete);
275    Delete desDelete = (Delete)Writables.getWritable(sb, new Delete());
276
277    assertTrue(Bytes.equals(delete.getRow(), desDelete.getRow()));
278    List<KeyValue> list = null;
279    List<KeyValue> desList = null;
280    for(Map.Entry<byte[], List<KeyValue>> entry :
281        delete.getFamilyMap().entrySet()){
282      assertTrue(desDelete.getFamilyMap().containsKey(entry.getKey()));
283      list = entry.getValue();
284      desList = desDelete.getFamilyMap().get(entry.getKey());
285      for(int i=0; i<list.size(); i++){
286        assertTrue(list.get(i).equals(desList.get(i)));
287      }
288    }
289  }
290  */
291
292  @Test public void testGet() throws Exception{
293    byte[] row = Bytes.toBytes("row");
294    byte[] fam = Bytes.toBytes("fam");
295    byte[] qf1 = Bytes.toBytes("qf1");
296
297    long ts = System.currentTimeMillis();
298    int maxVersions = 2;
299
300    Get get = new Get(row);
301    get.addColumn(fam, qf1);
302    get.setTimeRange(ts, ts+1);
303    get.readVersions(maxVersions);
304
305    ClientProtos.Get getProto = ProtobufUtil.toGet(get);
306    Get desGet = ProtobufUtil.toGet(getProto);
307
308    assertTrue(Bytes.equals(get.getRow(), desGet.getRow()));
309    Set<byte[]> set = null;
310    Set<byte[]> desSet = null;
311
312    for(Map.Entry<byte[], NavigableSet<byte[]>> entry :
313        get.getFamilyMap().entrySet()){
314      assertTrue(desGet.getFamilyMap().containsKey(entry.getKey()));
315      set = entry.getValue();
316      desSet = desGet.getFamilyMap().get(entry.getKey());
317      for(byte [] qualifier : set){
318        assertTrue(desSet.contains(qualifier));
319      }
320    }
321
322    assertEquals(get.getMaxVersions(), desGet.getMaxVersions());
323    TimeRange tr = get.getTimeRange();
324    TimeRange desTr = desGet.getTimeRange();
325    assertEquals(tr.getMax(), desTr.getMax());
326    assertEquals(tr.getMin(), desTr.getMin());
327  }
328
329
330  @Test public void testScan() throws Exception {
331
332    byte[] startRow = Bytes.toBytes("startRow");
333    byte[] stopRow  = Bytes.toBytes("stopRow");
334    byte[] fam = Bytes.toBytes("fam");
335    byte[] qf1 = Bytes.toBytes("qf1");
336
337    long ts = System.currentTimeMillis();
338    int maxVersions = 2;
339
340    Scan scan = new Scan(startRow, stopRow);
341    scan.addColumn(fam, qf1);
342    scan.setTimeRange(ts, ts+1);
343    scan.setMaxVersions(maxVersions);
344
345    ClientProtos.Scan scanProto = ProtobufUtil.toScan(scan);
346    Scan desScan = ProtobufUtil.toScan(scanProto);
347
348    assertTrue(Bytes.equals(scan.getStartRow(), desScan.getStartRow()));
349    assertTrue(Bytes.equals(scan.getStopRow(), desScan.getStopRow()));
350    assertEquals(scan.getCacheBlocks(), desScan.getCacheBlocks());
351    Set<byte[]> set = null;
352    Set<byte[]> desSet = null;
353
354    for(Map.Entry<byte[], NavigableSet<byte[]>> entry :
355        scan.getFamilyMap().entrySet()){
356      assertTrue(desScan.getFamilyMap().containsKey(entry.getKey()));
357      set = entry.getValue();
358      desSet = desScan.getFamilyMap().get(entry.getKey());
359      for(byte[] column : set){
360        assertTrue(desSet.contains(column));
361      }
362
363      // Test filters are serialized properly.
364      scan = new Scan(startRow);
365      final String name = "testScan";
366      byte [] prefix = Bytes.toBytes(name);
367      scan.setFilter(new PrefixFilter(prefix));
368      scanProto = ProtobufUtil.toScan(scan);
369      desScan = ProtobufUtil.toScan(scanProto);
370      Filter f = desScan.getFilter();
371      assertTrue(f instanceof PrefixFilter);
372    }
373
374    assertEquals(scan.getMaxVersions(), desScan.getMaxVersions());
375    TimeRange tr = scan.getTimeRange();
376    TimeRange desTr = desScan.getTimeRange();
377    assertEquals(tr.getMax(), desTr.getMax());
378    assertEquals(tr.getMin(), desTr.getMin());
379  }
380
381  /*
382   * TODO
383  @Test public void testResultEmpty() throws Exception {
384    List<KeyValue> keys = new ArrayList<KeyValue>();
385    Result r = Result.newResult(keys);
386    assertTrue(r.isEmpty());
387    byte [] rb = Writables.getBytes(r);
388    Result deserializedR = (Result)Writables.getWritable(rb, new Result());
389    assertTrue(deserializedR.isEmpty());
390  }
391
392
393  @Test public void testResult() throws Exception {
394    byte [] rowA = Bytes.toBytes("rowA");
395    byte [] famA = Bytes.toBytes("famA");
396    byte [] qfA = Bytes.toBytes("qfA");
397    byte [] valueA = Bytes.toBytes("valueA");
398
399    byte [] rowB = Bytes.toBytes("rowB");
400    byte [] famB = Bytes.toBytes("famB");
401    byte [] qfB = Bytes.toBytes("qfB");
402    byte [] valueB = Bytes.toBytes("valueB");
403
404    KeyValue kvA = new KeyValue(rowA, famA, qfA, valueA);
405    KeyValue kvB = new KeyValue(rowB, famB, qfB, valueB);
406
407    Result result = Result.newResult(new KeyValue[]{kvA, kvB});
408
409    byte [] rb = Writables.getBytes(result);
410    Result deResult = (Result)Writables.getWritable(rb, new Result());
411
412    assertTrue("results are not equivalent, first key mismatch",
413        result.raw()[0].equals(deResult.raw()[0]));
414
415    assertTrue("results are not equivalent, second key mismatch",
416        result.raw()[1].equals(deResult.raw()[1]));
417
418    // Test empty Result
419    Result r = new Result();
420    byte [] b = Writables.getBytes(r);
421    Result deserialized = (Result)Writables.getWritable(b, new Result());
422    assertEquals(r.size(), deserialized.size());
423  }
424
425  @Test public void testResultDynamicBuild() throws Exception {
426    byte [] rowA = Bytes.toBytes("rowA");
427    byte [] famA = Bytes.toBytes("famA");
428    byte [] qfA = Bytes.toBytes("qfA");
429    byte [] valueA = Bytes.toBytes("valueA");
430
431    byte [] rowB = Bytes.toBytes("rowB");
432    byte [] famB = Bytes.toBytes("famB");
433    byte [] qfB = Bytes.toBytes("qfB");
434    byte [] valueB = Bytes.toBytes("valueB");
435
436    KeyValue kvA = new KeyValue(rowA, famA, qfA, valueA);
437    KeyValue kvB = new KeyValue(rowB, famB, qfB, valueB);
438
439    Result result = Result.newResult(new KeyValue[]{kvA, kvB});
440
441    byte [] rb = Writables.getBytes(result);
442
443
444    // Call getRow() first
445    Result deResult = (Result)Writables.getWritable(rb, new Result());
446    byte [] row = deResult.getRow();
447    assertTrue(Bytes.equals(row, rowA));
448
449    // Call sorted() first
450    deResult = (Result)Writables.getWritable(rb, new Result());
451    assertTrue("results are not equivalent, first key mismatch",
452        result.raw()[0].equals(deResult.raw()[0]));
453    assertTrue("results are not equivalent, second key mismatch",
454        result.raw()[1].equals(deResult.raw()[1]));
455
456    // Call raw() first
457    deResult = (Result)Writables.getWritable(rb, new Result());
458    assertTrue("results are not equivalent, first key mismatch",
459        result.raw()[0].equals(deResult.raw()[0]));
460    assertTrue("results are not equivalent, second key mismatch",
461        result.raw()[1].equals(deResult.raw()[1]));
462
463
464  }
465
466  @Test public void testResultArray() throws Exception {
467    byte [] rowA = Bytes.toBytes("rowA");
468    byte [] famA = Bytes.toBytes("famA");
469    byte [] qfA = Bytes.toBytes("qfA");
470    byte [] valueA = Bytes.toBytes("valueA");
471
472    byte [] rowB = Bytes.toBytes("rowB");
473    byte [] famB = Bytes.toBytes("famB");
474    byte [] qfB = Bytes.toBytes("qfB");
475    byte [] valueB = Bytes.toBytes("valueB");
476
477    KeyValue kvA = new KeyValue(rowA, famA, qfA, valueA);
478    KeyValue kvB = new KeyValue(rowB, famB, qfB, valueB);
479
480
481    Result result1 = Result.newResult(new KeyValue[]{kvA, kvB});
482    Result result2 = Result.newResult(new KeyValue[]{kvB});
483    Result result3 = Result.newResult(new KeyValue[]{kvB});
484
485    Result [] results = new Result [] {result1, result2, result3};
486
487    ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
488    DataOutputStream out = new DataOutputStream(byteStream);
489    Result.writeArray(out, results);
490
491    byte [] rb = byteStream.toByteArray();
492
493    DataInputBuffer in = new DataInputBuffer();
494    in.reset(rb, 0, rb.length);
495
496    Result [] deResults = Result.readArray(in);
497
498    assertTrue(results.length == deResults.length);
499
500    for(int i=0;i<results.length;i++) {
501      KeyValue [] keysA = results[i].raw();
502      KeyValue [] keysB = deResults[i].raw();
503      assertTrue(keysA.length == keysB.length);
504      for(int j=0;j<keysA.length;j++) {
505        assertTrue("Expected equivalent keys but found:\n" +
506            "KeyA : " + keysA[j].toString() + "\n" +
507            "KeyB : " + keysB[j].toString() + "\n" +
508            keysA.length + " total keys, " + i + "th so far"
509            ,keysA[j].equals(keysB[j]));
510      }
511    }
512
513  }
514
515  @Test public void testResultArrayEmpty() throws Exception {
516    List<KeyValue> keys = new ArrayList<KeyValue>();
517    Result r = Result.newResult(keys);
518    Result [] results = new Result [] {r};
519
520    ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
521    DataOutputStream out = new DataOutputStream(byteStream);
522
523    Result.writeArray(out, results);
524
525    results = null;
526
527    byteStream = new ByteArrayOutputStream();
528    out = new DataOutputStream(byteStream);
529    Result.writeArray(out, results);
530
531    byte [] rb = byteStream.toByteArray();
532
533    DataInputBuffer in = new DataInputBuffer();
534    in.reset(rb, 0, rb.length);
535
536    Result [] deResults = Result.readArray(in);
537
538    assertTrue(deResults.length == 0);
539
540    results = new Result[0];
541
542    byteStream = new ByteArrayOutputStream();
543    out = new DataOutputStream(byteStream);
544    Result.writeArray(out, results);
545
546    rb = byteStream.toByteArray();
547
548    in = new DataInputBuffer();
549    in.reset(rb, 0, rb.length);
550
551    deResults = Result.readArray(in);
552
553    assertTrue(deResults.length == 0);
554
555  }
556  */
557
558  protected static final int MAXVERSIONS = 3;
559  protected final static byte [] fam1 = Bytes.toBytes("colfamily1");
560  protected final static byte [] fam2 = Bytes.toBytes("colfamily2");
561  protected final static byte [] fam3 = Bytes.toBytes("colfamily3");
562  protected static final byte [][] COLUMNS = {fam1, fam2, fam3};
563
564  /**
565   * Create a table of name <code>name</code> with {@link #COLUMNS} for
566   * families.
567   * @param name Name to give table.
568   * @return Column descriptor.
569   */
570  protected HTableDescriptor createTableDescriptor(final String name) {
571    return createTableDescriptor(name, MAXVERSIONS);
572  }
573
574  /**
575   * Create a table of name <code>name</code> with {@link #COLUMNS} for
576   * families.
577   * @param name Name to give table.
578   * @param versions How many versions to allow per column.
579   * @return Column descriptor.
580   */
581  protected HTableDescriptor createTableDescriptor(final String name,
582      final int versions) {
583    HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name));
584    htd.addFamily(new HColumnDescriptor(fam1)
585        .setMaxVersions(versions)
586        .setBlockCacheEnabled(false)
587    );
588    htd.addFamily(new HColumnDescriptor(fam2)
589        .setMaxVersions(versions)
590        .setBlockCacheEnabled(false)
591    );
592    htd.addFamily(new HColumnDescriptor(fam3)
593        .setMaxVersions(versions)
594        .setBlockCacheEnabled(false)
595    );
596    return htd;
597  }
598}