001/** 002 * 003 * Licensed to the Apache Software Foundation (ASF) under one 004 * or more contributor license agreements. See the NOTICE file 005 * distributed with this work for additional information 006 * regarding copyright ownership. The ASF licenses this file 007 * to you under the Apache License, Version 2.0 (the 008 * "License"); you may not use this file except in compliance 009 * with the License. You may obtain a copy of the License at 010 * 011 * http://www.apache.org/licenses/LICENSE-2.0 012 * 013 * Unless required by applicable law or agreed to in writing, software 014 * distributed under the License is distributed on an "AS IS" BASIS, 015 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 016 * See the License for the specific language governing permissions and 017 * limitations under the License. 018 */ 019package org.apache.hadoop.hbase.util; 020 021import static org.apache.hadoop.hbase.regionserver.HStoreFile.BULKLOAD_TIME_KEY; 022import static org.junit.Assert.assertArrayEquals; 023import static org.junit.Assert.fail; 024 025import java.io.IOException; 026import java.util.Arrays; 027import java.util.Locale; 028import java.util.Optional; 029 030import org.apache.hadoop.conf.Configuration; 031import org.apache.hadoop.fs.FileSystem; 032import org.apache.hadoop.fs.Path; 033import org.apache.hadoop.hbase.ArrayBackedTag; 034import org.apache.hadoop.hbase.Cell; 035import org.apache.hadoop.hbase.HColumnDescriptor; 036import org.apache.hadoop.hbase.KeyValue; 037import org.apache.hadoop.hbase.PrivateCellUtil; 038import org.apache.hadoop.hbase.Tag; 039import org.apache.hadoop.hbase.TagType; 040import org.apache.hadoop.hbase.client.Result; 041import org.apache.hadoop.hbase.client.ResultScanner; 042import org.apache.hadoop.hbase.client.Scan; 043import org.apache.hadoop.hbase.client.Table; 044import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; 045import org.apache.hadoop.hbase.io.hfile.CacheConfig; 046import org.apache.hadoop.hbase.io.hfile.HFile; 047import org.apache.hadoop.hbase.io.hfile.HFileContext; 048import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder; 049import org.apache.hadoop.hbase.mob.MobUtils; 050 051/** 052 * Utility class for HFile-related testing. 053 */ 054public class HFileTestUtil { 055 056 public static final String OPT_DATA_BLOCK_ENCODING_USAGE = 057 "Encoding algorithm (e.g. prefix " 058 + "compression) to use for data blocks in the test column family, " 059 + "one of " + Arrays.toString(DataBlockEncoding.values()) + "."; 060 public static final String OPT_DATA_BLOCK_ENCODING = 061 HColumnDescriptor.DATA_BLOCK_ENCODING.toLowerCase(Locale.ROOT); 062 /** Column family used by the test */ 063 public static byte[] DEFAULT_COLUMN_FAMILY = Bytes.toBytes("test_cf"); 064 /** Column families used by the test */ 065 public static final byte[][] DEFAULT_COLUMN_FAMILIES = { DEFAULT_COLUMN_FAMILY }; 066 067 /** 068 * Create an HFile with the given number of rows between a given 069 * start key and end key @ family:qualifier. The value will be the key value. 070 * This file will not have tags. 071 */ 072 public static void createHFile( 073 Configuration configuration, 074 FileSystem fs, Path path, 075 byte[] family, byte[] qualifier, 076 byte[] startKey, byte[] endKey, int numRows) throws IOException { 077 createHFile(configuration, fs, path, DataBlockEncoding.NONE, family, qualifier, 078 startKey, endKey, numRows, false); 079 } 080 081 /** 082 * Create an HFile with the given number of rows between a given 083 * start key and end key @ family:qualifier. The value will be the key value. 084 * This file will use certain data block encoding algorithm. 085 */ 086 public static void createHFileWithDataBlockEncoding( 087 Configuration configuration, 088 FileSystem fs, Path path, DataBlockEncoding encoding, 089 byte[] family, byte[] qualifier, 090 byte[] startKey, byte[] endKey, int numRows) throws IOException { 091 createHFile(configuration, fs, path, encoding, family, qualifier, startKey, endKey, 092 numRows, false); 093 } 094 095 /** 096 * Create an HFile with the given number of rows between a given 097 * start key and end key @ family:qualifier. The value will be the key value. 098 * This cells will also have a tag whose value is the key. 099 */ 100 public static void createHFileWithTags( 101 Configuration configuration, 102 FileSystem fs, Path path, 103 byte[] family, byte[] qualifier, 104 byte[] startKey, byte[] endKey, int numRows) throws IOException { 105 createHFile(configuration, fs, path, DataBlockEncoding.NONE, family, qualifier, 106 startKey, endKey, numRows, true); 107 } 108 109 /** 110 * Create an HFile with the given number of rows between a given 111 * start key and end key @ family:qualifier. 112 * If withTag is true, we add the rowKey as the tag value for 113 * tagtype MOB_TABLE_NAME_TAG_TYPE 114 */ 115 public static void createHFile( 116 Configuration configuration, 117 FileSystem fs, Path path, DataBlockEncoding encoding, 118 byte[] family, byte[] qualifier, 119 byte[] startKey, byte[] endKey, int numRows, boolean withTag) throws IOException { 120 HFileContext meta = new HFileContextBuilder() 121 .withIncludesTags(withTag) 122 .withDataBlockEncoding(encoding) 123 .build(); 124 HFile.Writer writer = HFile.getWriterFactory(configuration, new CacheConfig(configuration)) 125 .withPath(fs, path) 126 .withFileContext(meta) 127 .create(); 128 long now = System.currentTimeMillis(); 129 try { 130 // subtract 2 since iterateOnSplits doesn't include boundary keys 131 for (byte[] key : Bytes.iterateOnSplits(startKey, endKey, numRows - 2)) { 132 Cell kv = new KeyValue(key, family, qualifier, now, key); 133 if (withTag) { 134 // add a tag. Arbitrarily chose mob tag since we have a helper already. 135 Tag tableNameTag = new ArrayBackedTag(TagType.MOB_TABLE_NAME_TAG_TYPE, key); 136 kv = MobUtils.createMobRefCell(kv, key, tableNameTag); 137 138 // verify that the kv has the tag. 139 Optional<Tag> tag = PrivateCellUtil.getTag(kv, TagType.MOB_TABLE_NAME_TAG_TYPE); 140 if (!tag.isPresent()) { 141 throw new IllegalStateException("Tag didn't stick to KV " + kv.toString()); 142 } 143 } 144 writer.append(kv); 145 } 146 } finally { 147 writer.appendFileInfo(BULKLOAD_TIME_KEY, Bytes.toBytes(System.currentTimeMillis())); 148 writer.close(); 149 } 150 } 151 152 /** 153 * This verifies that each cell has a tag that is equal to its rowkey name. For this to work 154 * the hbase instance must have HConstants.RPC_CODEC_CONF_KEY set to 155 * KeyValueCodecWithTags.class.getCanonicalName()); 156 * @param table table containing tagged cells 157 * @throws IOException if problems reading table 158 */ 159 public static void verifyTags(Table table) throws IOException { 160 ResultScanner s = table.getScanner(new Scan()); 161 for (Result r : s) { 162 for (Cell c : r.listCells()) { 163 Optional<Tag> tag = PrivateCellUtil.getTag(c, TagType.MOB_TABLE_NAME_TAG_TYPE); 164 if (!tag.isPresent()) { 165 fail(c.toString() + " has null tag"); 166 continue; 167 } 168 Tag t = tag.get(); 169 byte[] tval = Tag.cloneValue(t); 170 assertArrayEquals(c.toString() + " has tag" + Bytes.toString(tval), 171 r.getRow(), tval); 172 } 173 } 174 } 175}