001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.filter; 019 020import java.io.IOException; 021import java.util.ArrayList; 022import java.util.Arrays; 023import java.util.List; 024import org.apache.commons.codec.binary.Hex; 025import org.apache.hadoop.hbase.HBaseClassTestRule; 026import org.apache.hadoop.hbase.TableName; 027import org.apache.hadoop.hbase.client.Put; 028import org.apache.hadoop.hbase.client.Result; 029import org.apache.hadoop.hbase.client.ResultScanner; 030import org.apache.hadoop.hbase.client.Scan; 031import org.apache.hadoop.hbase.client.Table; 032import org.apache.hadoop.hbase.testclassification.FilterTests; 033import org.apache.hadoop.hbase.testclassification.MediumTests; 034import org.junit.Assert; 035import org.junit.ClassRule; 036import org.junit.Rule; 037import org.junit.Test; 038import org.junit.experimental.categories.Category; 039import org.junit.rules.TestName; 040import org.slf4j.Logger; 041import org.slf4j.LoggerFactory; 042 043/** 044 * Test if Scan.setRowPrefixFilter works as intended. 045 */ 046@Category({FilterTests.class, MediumTests.class}) 047public class TestScanRowPrefix extends FilterTestingCluster { 048 049 @ClassRule 050 public static final HBaseClassTestRule CLASS_RULE = 051 HBaseClassTestRule.forClass(TestScanRowPrefix.class); 052 053 private static final Logger LOG = LoggerFactory 054 .getLogger(TestScanRowPrefix.class); 055 056 @Rule 057 public TestName name = new TestName(); 058 059 @Test 060 public void testPrefixScanning() throws IOException { 061 final TableName tableName = TableName.valueOf(name.getMethodName()); 062 createTable(tableName,"F"); 063 Table table = openTable(tableName); 064 065 /** 066 * Note that about half of these tests were relevant for an different implementation approach 067 * of setRowPrefixFilter. These test cases have been retained to ensure that also the 068 * edge cases found there are still covered. 069 */ 070 071 final byte[][] rowIds = { 072 {(byte) 0x11}, // 0 073 {(byte) 0x12}, // 1 074 {(byte) 0x12, (byte) 0x23, (byte) 0xFF, (byte) 0xFE}, // 2 075 {(byte) 0x12, (byte) 0x23, (byte) 0xFF, (byte) 0xFF}, // 3 076 {(byte) 0x12, (byte) 0x23, (byte) 0xFF, (byte) 0xFF, (byte) 0x00}, // 4 077 {(byte) 0x12, (byte) 0x23, (byte) 0xFF, (byte) 0xFF, (byte) 0x01}, // 5 078 {(byte) 0x12, (byte) 0x24}, // 6 079 {(byte) 0x12, (byte) 0x24, (byte) 0x00}, // 7 080 {(byte) 0x12, (byte) 0x24, (byte) 0x00, (byte) 0x00}, // 8 081 {(byte) 0x12, (byte) 0x25}, // 9 082 {(byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF}, // 10 083 }; 084 for (byte[] rowId: rowIds) { 085 Put p = new Put(rowId); 086 // Use the rowId as the column qualifier 087 p.addColumn("F".getBytes(), rowId, "Dummy value".getBytes()); 088 table.put(p); 089 } 090 091 byte[] prefix0 = {}; 092 List<byte[]> expected0 = new ArrayList<>(16); 093 expected0.addAll(Arrays.asList(rowIds)); // Expect all rows 094 095 byte[] prefix1 = {(byte) 0x12, (byte) 0x23}; 096 List<byte[]> expected1 = new ArrayList<>(16); 097 expected1.add(rowIds[2]); 098 expected1.add(rowIds[3]); 099 expected1.add(rowIds[4]); 100 expected1.add(rowIds[5]); 101 102 byte[] prefix2 = {(byte) 0x12, (byte) 0x23, (byte) 0xFF, (byte) 0xFF}; 103 List<byte[]> expected2 = new ArrayList<>(); 104 expected2.add(rowIds[3]); 105 expected2.add(rowIds[4]); 106 expected2.add(rowIds[5]); 107 108 byte[] prefix3 = {(byte) 0x12, (byte) 0x24}; 109 List<byte[]> expected3 = new ArrayList<>(); 110 expected3.add(rowIds[6]); 111 expected3.add(rowIds[7]); 112 expected3.add(rowIds[8]); 113 114 byte[] prefix4 = {(byte) 0xFF, (byte) 0xFF}; 115 List<byte[]> expected4 = new ArrayList<>(); 116 expected4.add(rowIds[10]); 117 118 // ======== 119 // PREFIX 0 120 Scan scan = new Scan(); 121 scan.setRowPrefixFilter(prefix0); 122 verifyScanResult(table, scan, expected0, "Scan empty prefix failed"); 123 124 // ======== 125 // PREFIX 1 126 scan = new Scan(); 127 scan.setRowPrefixFilter(prefix1); 128 verifyScanResult(table, scan, expected1, "Scan normal prefix failed"); 129 130 scan.setRowPrefixFilter(null); 131 verifyScanResult(table, scan, expected0, "Scan after prefix reset failed"); 132 133 scan = new Scan(); 134 scan.setFilter(new ColumnPrefixFilter(prefix1)); 135 verifyScanResult(table, scan, expected1, "Double check on column prefix failed"); 136 137 // ======== 138 // PREFIX 2 139 scan = new Scan(); 140 scan.setRowPrefixFilter(prefix2); 141 verifyScanResult(table, scan, expected2, "Scan edge 0xFF prefix failed"); 142 143 scan.setRowPrefixFilter(null); 144 verifyScanResult(table, scan, expected0, "Scan after prefix reset failed"); 145 146 scan = new Scan(); 147 scan.setFilter(new ColumnPrefixFilter(prefix2)); 148 verifyScanResult(table, scan, expected2, "Double check on column prefix failed"); 149 150 // ======== 151 // PREFIX 3 152 scan = new Scan(); 153 scan.setRowPrefixFilter(prefix3); 154 verifyScanResult(table, scan, expected3, "Scan normal with 0x00 ends failed"); 155 156 scan.setRowPrefixFilter(null); 157 verifyScanResult(table, scan, expected0, "Scan after prefix reset failed"); 158 159 scan = new Scan(); 160 scan.setFilter(new ColumnPrefixFilter(prefix3)); 161 verifyScanResult(table, scan, expected3, "Double check on column prefix failed"); 162 163 // ======== 164 // PREFIX 4 165 scan = new Scan(); 166 scan.setRowPrefixFilter(prefix4); 167 verifyScanResult(table, scan, expected4, "Scan end prefix failed"); 168 169 scan.setRowPrefixFilter(null); 170 verifyScanResult(table, scan, expected0, "Scan after prefix reset failed"); 171 172 scan = new Scan(); 173 scan.setFilter(new ColumnPrefixFilter(prefix4)); 174 verifyScanResult(table, scan, expected4, "Double check on column prefix failed"); 175 176 // ======== 177 // COMBINED 178 // Prefix + Filter 179 scan = new Scan(); 180 scan.setRowPrefixFilter(prefix1); 181 verifyScanResult(table, scan, expected1, "Prefix filter failed"); 182 183 scan.setFilter(new ColumnPrefixFilter(prefix2)); 184 verifyScanResult(table, scan, expected2, "Combined Prefix + Filter failed"); 185 186 scan.setRowPrefixFilter(null); 187 verifyScanResult(table, scan, expected2, "Combined Prefix + Filter; removing Prefix failed"); 188 189 scan.setFilter(null); 190 verifyScanResult(table, scan, expected0, "Scan after Filter reset failed"); 191 192 // ======== 193 // Reversed: Filter + Prefix 194 scan = new Scan(); 195 scan.setFilter(new ColumnPrefixFilter(prefix2)); 196 verifyScanResult(table, scan, expected2, "Test filter failed"); 197 198 scan.setRowPrefixFilter(prefix1); 199 verifyScanResult(table, scan, expected2, "Combined Filter + Prefix failed"); 200 201 scan.setFilter(null); 202 verifyScanResult(table, scan, expected1, "Combined Filter + Prefix ; removing Filter failed"); 203 204 scan.setRowPrefixFilter(null); 205 verifyScanResult(table, scan, expected0, "Scan after prefix reset failed"); 206 } 207 208 private void verifyScanResult(Table table, Scan scan, List<byte[]> expectedKeys, String message) { 209 List<byte[]> actualKeys = new ArrayList<>(); 210 try { 211 ResultScanner scanner = table.getScanner(scan); 212 for (Result result : scanner) { 213 actualKeys.add(result.getRow()); 214 } 215 216 String fullMessage = message; 217 if (LOG.isDebugEnabled()) { 218 fullMessage = message + "\n" + tableOfTwoListsOfByteArrays( 219 "Expected", expectedKeys, 220 "Actual ", actualKeys); 221 } 222 223 Assert.assertArrayEquals( 224 fullMessage, 225 expectedKeys.toArray(), 226 actualKeys.toArray()); 227 } catch (IOException e) { 228 e.printStackTrace(); 229 Assert.fail(); 230 } 231 } 232 233 private String printMultiple(char letter, int count) { 234 StringBuilder sb = new StringBuilder(count); 235 for (int i = 0; i < count; i++) { 236 sb.append(letter); 237 } 238 return sb.toString(); 239 } 240 241 private String tableOfTwoListsOfByteArrays( 242 String label1, List<byte[]> listOfBytes1, 243 String label2, List<byte[]> listOfBytes2) { 244 int margin1 = calculateWidth(label1, listOfBytes1); 245 int margin2 = calculateWidth(label2, listOfBytes2); 246 247 StringBuilder sb = new StringBuilder(512); 248 String separator = '+' + printMultiple('-', margin1 + margin2 + 5) + '+' + '\n'; 249 sb.append(separator); 250 sb.append(printLine(label1, margin1, label2, margin2)).append('\n'); 251 sb.append(separator); 252 int maxLength = Math.max(listOfBytes1.size(), listOfBytes2.size()); 253 for (int offset = 0; offset < maxLength; offset++) { 254 String value1 = getStringFromList(listOfBytes1, offset); 255 String value2 = getStringFromList(listOfBytes2, offset); 256 sb.append(printLine(value1, margin1, value2, margin2)).append('\n'); 257 } 258 sb.append(separator).append('\n'); 259 return sb.toString(); 260 } 261 262 private String printLine(String leftValue, int leftWidth1, String rightValue, int rightWidth) { 263 return "| " + 264 leftValue + printMultiple(' ', leftWidth1 - leftValue.length() ) + 265 " | " + 266 rightValue + printMultiple(' ', rightWidth - rightValue.length()) + 267 " |"; 268 } 269 270 private int calculateWidth(String label1, List<byte[]> listOfBytes1) { 271 int longestList1 = label1.length(); 272 for (byte[] value : listOfBytes1) { 273 longestList1 = Math.max(value.length * 2, longestList1); 274 } 275 return longestList1 + 5; 276 } 277 278 private String getStringFromList(List<byte[]> listOfBytes, int offset) { 279 String value1; 280 if (listOfBytes.size() > offset) { 281 value1 = Hex.encodeHexString(listOfBytes.get(offset)); 282 } else { 283 value1 = "<missing>"; 284 } 285 return value1; 286 } 287 288}