001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase; 019 020import static org.junit.Assert.assertEquals; 021import static org.junit.Assert.assertNotNull; 022import static org.junit.Assert.assertTrue; 023import static org.junit.Assert.fail; 024 025import java.io.ByteArrayInputStream; 026import java.io.ByteArrayOutputStream; 027import java.io.DataInputStream; 028import java.io.DataOutputStream; 029import java.io.IOException; 030import java.util.List; 031import java.util.Map; 032import java.util.NavigableSet; 033import java.util.Set; 034import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; 035import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; 036import org.apache.hadoop.hbase.client.Get; 037import org.apache.hadoop.hbase.client.RegionInfo; 038import org.apache.hadoop.hbase.client.RegionInfoBuilder; 039import org.apache.hadoop.hbase.client.Scan; 040import org.apache.hadoop.hbase.client.TableDescriptor; 041import org.apache.hadoop.hbase.client.TableDescriptorBuilder; 042import org.apache.hadoop.hbase.filter.BinaryComparator; 043import org.apache.hadoop.hbase.filter.Filter; 044import org.apache.hadoop.hbase.filter.PrefixFilter; 045import org.apache.hadoop.hbase.filter.RowFilter; 046import org.apache.hadoop.hbase.io.TimeRange; 047import org.apache.hadoop.hbase.testclassification.MiscTests; 048import org.apache.hadoop.hbase.testclassification.SmallTests; 049import org.apache.hadoop.hbase.util.Bytes; 050import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; 051import org.apache.hadoop.io.DataInputBuffer; 052import org.junit.ClassRule; 053import org.junit.Test; 054import org.junit.experimental.categories.Category; 055 056import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; 057import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos; 058 059/** 060 * Test HBase Writables serializations 061 */ 062@Category({ MiscTests.class, SmallTests.class }) 063public class TestSerialization { 064 065 @ClassRule 066 public static final HBaseClassTestRule CLASS_RULE = 067 HBaseClassTestRule.forClass(TestSerialization.class); 068 069 @Test 070 public void testKeyValue() throws Exception { 071 final String name = "testKeyValue2"; 072 byte[] row = Bytes.toBytes(name); 073 byte[] fam = Bytes.toBytes("fam"); 074 byte[] qf = Bytes.toBytes("qf"); 075 long ts = EnvironmentEdgeManager.currentTime(); 076 byte[] val = Bytes.toBytes("val"); 077 KeyValue kv = new KeyValue(row, fam, qf, ts, val); 078 ByteArrayOutputStream baos = new ByteArrayOutputStream(); 079 DataOutputStream dos = new DataOutputStream(baos); 080 KeyValueUtil.write(kv, dos); 081 dos.close(); 082 byte[] mb = baos.toByteArray(); 083 ByteArrayInputStream bais = new ByteArrayInputStream(mb); 084 DataInputStream dis = new DataInputStream(bais); 085 KeyValue deserializedKv = KeyValueUtil.create(dis); 086 assertTrue(Bytes.equals(kv.getBuffer(), deserializedKv.getBuffer())); 087 assertEquals(kv.getOffset(), deserializedKv.getOffset()); 088 assertEquals(kv.getLength(), deserializedKv.getLength()); 089 } 090 091 @Test 092 public void testCreateKeyValueInvalidNegativeLength() { 093 094 KeyValue kv_0 = new KeyValue(Bytes.toBytes("myRow"), Bytes.toBytes("myCF"), // 51 bytes 095 Bytes.toBytes("myQualifier"), 12345L, Bytes.toBytes("my12345")); 096 097 KeyValue kv_1 = new KeyValue(Bytes.toBytes("myRow"), Bytes.toBytes("myCF"), // 49 bytes 098 Bytes.toBytes("myQualifier"), 12345L, Bytes.toBytes("my123")); 099 100 ByteArrayOutputStream baos = new ByteArrayOutputStream(); 101 DataOutputStream dos = new DataOutputStream(baos); 102 103 long l = 0; 104 try { 105 l = KeyValue.oswrite(kv_0, dos, false); 106 l += KeyValue.oswrite(kv_1, dos, false); 107 assertEquals(100L, l); 108 } catch (IOException e) { 109 fail("Unexpected IOException" + e.getMessage()); 110 } 111 112 ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray()); 113 DataInputStream dis = new DataInputStream(bais); 114 115 try { 116 KeyValueUtil.create(dis); 117 assertTrue(kv_0.equals(kv_1)); 118 } catch (Exception e) { 119 fail("Unexpected Exception" + e.getMessage()); 120 } 121 122 // length -1 123 try { 124 // even if we have a good kv now in dis we will just pass length with -1 for simplicity 125 KeyValueUtil.create(-1, dis); 126 fail("Expected corrupt stream"); 127 } catch (Exception e) { 128 assertEquals("Failed read -1 bytes, stream corrupt?", e.getMessage()); 129 } 130 131 } 132 133 @Test 134 public void testCompareFilter() throws Exception { 135 Filter f = 136 new RowFilter(CompareOperator.EQUAL, new BinaryComparator(Bytes.toBytes("testRowOne-2"))); 137 byte[] bytes = f.toByteArray(); 138 Filter ff = RowFilter.parseFrom(bytes); 139 assertNotNull(ff); 140 } 141 142 @Test 143 public void testTableDescriptor() throws Exception { 144 final String name = "testTableDescriptor"; 145 TableDescriptor htd = createTableDescriptor(name); 146 byte[] mb = TableDescriptorBuilder.toByteArray(htd); 147 TableDescriptor deserializedHtd = TableDescriptorBuilder.parseFrom(mb); 148 assertEquals(htd.getTableName(), deserializedHtd.getTableName()); 149 } 150 151 /** 152 * Test RegionInfo serialization 153 */ 154 @Test 155 public void testRegionInfo() throws Exception { 156 RegionInfo hri = createRandomRegion("testRegionInfo"); 157 158 // test toByteArray() 159 byte[] hrib = RegionInfo.toByteArray(hri); 160 RegionInfo deserializedHri = RegionInfo.parseFrom(hrib); 161 assertEquals(hri.getEncodedName(), deserializedHri.getEncodedName()); 162 assertEquals(hri, deserializedHri); 163 164 // test toDelimitedByteArray() 165 hrib = RegionInfo.toDelimitedByteArray(hri); 166 DataInputBuffer buf = new DataInputBuffer(); 167 try { 168 buf.reset(hrib, hrib.length); 169 deserializedHri = RegionInfo.parseFrom(buf); 170 assertEquals(hri.getEncodedName(), deserializedHri.getEncodedName()); 171 assertEquals(hri, deserializedHri); 172 } finally { 173 buf.close(); 174 } 175 } 176 177 @Test 178 public void testRegionInfos() throws Exception { 179 RegionInfo hri = createRandomRegion("testRegionInfos"); 180 byte[] triple = RegionInfo.toDelimitedByteArray(hri, hri, hri); 181 List<RegionInfo> regions = RegionInfo.parseDelimitedFrom(triple, 0, triple.length); 182 assertTrue(regions.size() == 3); 183 assertTrue(regions.get(0).equals(regions.get(1))); 184 assertTrue(regions.get(0).equals(regions.get(2))); 185 } 186 187 private RegionInfo createRandomRegion(final String name) { 188 TableDescriptorBuilder tableDescriptorBuilder = 189 TableDescriptorBuilder.newBuilder(TableName.valueOf(name)); 190 String[] families = new String[] { "info", "anchor" }; 191 for (int i = 0; i < families.length; i++) { 192 ColumnFamilyDescriptor columnFamilyDescriptor = 193 ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(families[i])).build(); 194 tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor); 195 } 196 TableDescriptor tableDescriptor = tableDescriptorBuilder.build(); 197 return RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build(); 198 } 199 200 @Test 201 public void testGet() throws Exception { 202 byte[] row = Bytes.toBytes("row"); 203 byte[] fam = Bytes.toBytes("fam"); 204 byte[] qf1 = Bytes.toBytes("qf1"); 205 long ts = EnvironmentEdgeManager.currentTime(); 206 int maxVersions = 2; 207 208 Get get = new Get(row); 209 get.addColumn(fam, qf1); 210 get.setTimeRange(ts, ts + 1); 211 get.readVersions(maxVersions); 212 213 ClientProtos.Get getProto = ProtobufUtil.toGet(get); 214 Get desGet = ProtobufUtil.toGet(getProto); 215 216 assertTrue(Bytes.equals(get.getRow(), desGet.getRow())); 217 Set<byte[]> set = null; 218 Set<byte[]> desSet = null; 219 220 for (Map.Entry<byte[], NavigableSet<byte[]>> entry : get.getFamilyMap().entrySet()) { 221 assertTrue(desGet.getFamilyMap().containsKey(entry.getKey())); 222 set = entry.getValue(); 223 desSet = desGet.getFamilyMap().get(entry.getKey()); 224 for (byte[] qualifier : set) { 225 assertTrue(desSet.contains(qualifier)); 226 } 227 } 228 229 assertEquals(get.getMaxVersions(), desGet.getMaxVersions()); 230 TimeRange tr = get.getTimeRange(); 231 TimeRange desTr = desGet.getTimeRange(); 232 assertEquals(tr.getMax(), desTr.getMax()); 233 assertEquals(tr.getMin(), desTr.getMin()); 234 } 235 236 @Test 237 public void testScan() throws Exception { 238 239 byte[] startRow = Bytes.toBytes("startRow"); 240 byte[] stopRow = Bytes.toBytes("stopRow"); 241 byte[] fam = Bytes.toBytes("fam"); 242 byte[] qf1 = Bytes.toBytes("qf1"); 243 long ts = EnvironmentEdgeManager.currentTime(); 244 int maxVersions = 2; 245 246 Scan scan = new Scan().withStartRow(startRow).withStopRow(stopRow); 247 scan.addColumn(fam, qf1); 248 scan.setTimeRange(ts, ts + 1); 249 scan.readVersions(maxVersions); 250 251 ClientProtos.Scan scanProto = ProtobufUtil.toScan(scan); 252 Scan desScan = ProtobufUtil.toScan(scanProto); 253 254 assertTrue(Bytes.equals(scan.getStartRow(), desScan.getStartRow())); 255 assertTrue(Bytes.equals(scan.getStopRow(), desScan.getStopRow())); 256 assertEquals(scan.getCacheBlocks(), desScan.getCacheBlocks()); 257 Set<byte[]> set = null; 258 Set<byte[]> desSet = null; 259 260 for (Map.Entry<byte[], NavigableSet<byte[]>> entry : scan.getFamilyMap().entrySet()) { 261 assertTrue(desScan.getFamilyMap().containsKey(entry.getKey())); 262 set = entry.getValue(); 263 desSet = desScan.getFamilyMap().get(entry.getKey()); 264 for (byte[] column : set) { 265 assertTrue(desSet.contains(column)); 266 } 267 268 // Test filters are serialized properly. 269 scan = new Scan().withStartRow(startRow); 270 final String name = "testScan"; 271 byte[] prefix = Bytes.toBytes(name); 272 scan.setFilter(new PrefixFilter(prefix)); 273 scanProto = ProtobufUtil.toScan(scan); 274 desScan = ProtobufUtil.toScan(scanProto); 275 Filter f = desScan.getFilter(); 276 assertTrue(f instanceof PrefixFilter); 277 } 278 279 assertEquals(scan.getMaxVersions(), desScan.getMaxVersions()); 280 TimeRange tr = scan.getTimeRange(); 281 TimeRange desTr = desScan.getTimeRange(); 282 assertEquals(tr.getMax(), desTr.getMax()); 283 assertEquals(tr.getMin(), desTr.getMin()); 284 } 285 286 protected static final int MAXVERSIONS = 3; 287 protected final static byte[] fam1 = Bytes.toBytes("colfamily1"); 288 protected final static byte[] fam2 = Bytes.toBytes("colfamily2"); 289 protected final static byte[] fam3 = Bytes.toBytes("colfamily3"); 290 protected static final byte[][] COLUMNS = { fam1, fam2, fam3 }; 291 292 /** 293 * Create a table of name <code>name</code> with {@link #COLUMNS} for families. 294 * @param name Name to give table. 295 * @return Column descriptor. 296 */ 297 protected TableDescriptor createTableDescriptor(final String name) { 298 return createTableDescriptor(name, MAXVERSIONS); 299 } 300 301 /** 302 * Create a table of name <code>name</code> with {@link #COLUMNS} for families. 303 * @param name Name to give table. 304 * @param versions How many versions to allow per column. 305 * @return Column descriptor. 306 */ 307 protected TableDescriptor createTableDescriptor(final String name, final int versions) { 308 TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(TableName.valueOf(name)); 309 builder 310 .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(fam1).setMaxVersions(versions) 311 .setBlockCacheEnabled(false).build()) 312 .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(fam2).setMaxVersions(versions) 313 .setBlockCacheEnabled(false).build()) 314 .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(fam3).setMaxVersions(versions) 315 .setBlockCacheEnabled(false).build()); 316 return builder.build(); 317 } 318}